多头注意力实现
2025/10/16大约 2 分钟
多头注意力实现MHA
import math
import torch
from torch import nn
class MultiHeadAttention(nn.Module):
'''
GPT 多头注意力实现
'''
def __init__(self, num_head, d_model, dropout=0.1):
super().__init__()
self.num_head = num_head
self.d_model = d_model
self.d_k = d_model // num_head
self.W_query = nn.Linear(self.d_model, self.d_model)
self.W_key = nn.Linear(self.d_model, self.d_model)
self.W_value = nn.Linear(self.d_model, self.d_model)
self.dropout = nn.Dropout(dropout)
self.out_proj = nn.Linear(self.d_model, self.d_model)
def forward(self, x, causal_mask=True, padding_mask=None):
# 1. 获取输入维度
batch_size, seq_len, _ = x.shape
# 2. 生成q,k,v
query = self.W_query(x).view(batch_size, seq_len, self.num_head, self.d_k).transpose(1, 2)
key = self.W_key(x).view(batch_size, seq_len, self.num_head, self.d_k).transpose(1, 2)
value = self.W_value(x).view(batch_size, seq_len, self.num_head, self.d_k).transpose(1, 2)
# 3. 生成自注意力分数
scores = torch.matmul(query, key.transpose(-1, -2)) / math.sqrt(self.d_k)
# 4. 叠加因果掩码
if causal_mask:
mask = torch.ones(batch_size, self.num_head, seq_len, seq_len, device=x.device)
mask = torch.triu(mask, diagonal=1)
scores = scores.masked_fill(mask == 1, float('-inf'))
# 5. 叠加填充掩码
if padding_mask is not None:
padding_mask = padding_mask.unsqueeze(1).unsqueeze(2)
scores = scores.masked_fill(padding_mask, float('-inf'))
# 6. 计算自注意力权重
att_weight = scores.softmax(dim=-1)
# 7. 随机失活 -> 自注意力 -> 整理维度
att_weight = self.dropout(att_weight)
attention = torch.matmul(att_weight, value)
attention = attention.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
# 8. 投影
out = self.out_proj(attention)
return out
if __name__ == '__main__':
x = torch.rand(3, 4, 768)
mha = MultiHeadAttention(12, 768)
padding_mask = torch.tensor([[False, False, False, True],
[False, False, True, True],
[False, True, True, True]], device=x.device)
print(mha(x, causal_mask=True, padding_mask=padding_mask).shape)分组查询注意力GQA
import math
import torch
from torch import nn
class GroupQueryAttention(nn.Module):
def __init__(self, num_head, d_model, num_query_groups, dropout=0.1):
super().__init__()
self.num_head = num_head
self.d_model = d_model
self.d_k = d_model // num_head
self.num_query_groups = num_query_groups
self.heads_per_group = self.num_head // self.num_query_groups
self.W_q = nn.Linear(d_model, self.num_head * self.d_k)
self.W_k = nn.Linear(d_model, self.num_query_groups * self.d_k)
self.W_v = nn.Linear(d_model, self.num_query_groups * self.d_k)
self.dropout = nn.Dropout(dropout)
self.out_proj = nn.Linear(d_model, d_model)
def forward(self, x, causal_mask=True, padding_mask=None):
# 1. 获取输入维度
batch_size, seq_len, _ = x.shape
# 2. 生成q, k, v
query = self.W_q(x).view(batch_size, seq_len, self.num_head, self.d_k).transpose(1, 2)
key = self.W_k(x).view(batch_size, seq_len, self.num_query_groups, self.d_k).transpose(1, 2)
value = self.W_v(x).view(batch_size, seq_len, self.num_query_groups, self.d_k).transpose(1, 2)
# 3. 扩展k, v
key = key.repeat_interleave(self.heads_per_group, dim=1)
value = value.repeat_interleave(self.heads_per_group, dim=1)
# 4. 计算注意力分数
scores = torch.matmul(query, key.transpose(-1, -2)) / math.sqrt(self.d_k)
# 5. 叠加因果掩码
if causal_mask:
mask = torch.ones(batch_size, self.num_head, seq_len, seq_len, device=x.device)
mask = torch.triu(mask, diagonal=1)
scores = scores.masked_fill(mask == 1, float('-inf'))
# 6. 叠加填充掩码
if padding_mask is not None:
padding_mask = padding_mask.unsqueeze(1).unsqueeze(2)
scores = scores.masked_fill(padding_mask, float('-inf'))
# 7. 计算注意力权重
att_weight = torch.softmax(scores, dim=-1)
# 8. 随机失活 -> 计算注意力 -> 整理维度
att_weight = self.dropout(att_weight)
attention = torch.matmul(att_weight, value)
attention = attention.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
# 9. 最终投影
output = self.out_proj(attention)
return output
if __name__ == '__main__':
x = torch.rand(3, 4, 768)
gqa = GroupQueryAttention(12, 768, 4)
padding_mask = torch.tensor([[False, False, False, True],
[False, False, True, True],
[False, True, True, True]], device=x.device)
print(gqa(x, True, padding_mask).shape)