多头注意力实现MHA
import math
import torch
from torch import nn
class MultiHeadAttention(nn.Module):
'''
GPT 多头注意力实现
'''
def __init__(self, num_head, d_model, dropout=0.1):
super().__init__()
self.num_head = num_head
self.d_model = d_model
self.d_k = d_model // num_head
self.W_query = nn.Linear(self.d_model, self.d_model)
self.W_key = nn.Linear(self.d_model, self.d_model)
self.W_value = nn.Linear(self.d_model, self.d_model)
self.dropout = nn.Dropout(dropout)
self.out_proj = nn.Linear(self.d_model, self.d_model)
def forward(self, x, causal_mask=True, padding_mask=None):
# 1. 获取输入维度
batch_size, seq_len, _ = x.shape
# 2. 生成q,k,v
query = self.W_query(x).view(batch_size, seq_len, self.num_head, self.d_k).transpose(1, 2)
key = self.W_key(x).view(batch_size, seq_len, self.num_head, self.d_k).transpose(1, 2)
value = self.W_value(x).view(batch_size, seq_len, self.num_head, self.d_k).transpose(1, 2)
# 3. 生成自注意力分数
scores = torch.matmul(query, key.transpose(-1, -2)) / math.sqrt(self.d_k)
# 4. 叠加因果掩码
if causal_mask:
mask = torch.ones(batch_size, self.num_head, seq_len, seq_len, device=x.device)
mask = torch.triu(mask, diagonal=1)
scores = scores.masked_fill(mask == 1, float('-inf'))
# 5. 叠加填充掩码
if padding_mask is not None:
padding_mask = padding_mask.unsqueeze(1).unsqueeze(2)
scores = scores.masked_fill(padding_mask, float('-inf'))
# 6. 计算自注意力权重
att_weight = scores.softmax(dim=-1)
# 7. 随机失活 -> 自注意力 -> 整理维度
att_weight = self.dropout(att_weight)
attention = torch.matmul(att_weight, value)
attention = attention.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
# 8. 投影
out = self.out_proj(attention)
return out
if __name__ == '__main__':
x = torch.rand(3, 4, 768)
mha = MultiHeadAttention(12, 768)
padding_mask = torch.tensor([[False, False, False, True],
[False, False, True, True],
[False, True, True, True]], device=x.device)
print(mha(x, causal_mask=True, padding_mask=padding_mask).shape)
2025/10/16大约 2 分钟