※ 引述《wu10200512 (廷廷)》之銘言：
: 老師要我用純pytorch寫transformer
: 我寫不出來
: 有沒有人會寫可以救我
: 拜託 我求你了
當你想要使用PyTorch來實現Transformer模型時，下面是一個基本的範例，演示如何使用
PyTorch建構Transformer模型。
首先，你需要匯入所需的PyTorch模組：
import torch
import torch.nn as nn
import torch.nn.functional as F
接下來，定義一個Transformer模型的類別：
class Transformer(nn.Module):
def __init__(self, input_dim, hidden_dim, num_heads, num_layers):
super(Transformer, self).__init__()
self.embedding = nn.Embedding(input_dim, hidden_dim)
self.positional_encoding = PositionalEncoding(hidden_dim)
self.encoder = Encoder(hidden_dim, num_heads, num_layers)
def forward(self, x):
embedded = self.embedding(x)
encoded = self.positional_encoding(embedded)
output = self.encoder(encoded)
return output
在這個範例中，我們使用了三個子模組：Embedding、PositionalEncoding和Encoder。這
些子模組可以透過定義相應的類別來實現。
class PositionalEncoding(nn.Module):
def __init__(self, hidden_dim, max_len=1000):
super(PositionalEncoding, self).__init__()
self.hidden_dim = hidden_dim
position = torch.arange(0, max_len).unsqueeze(1)
div_term = torch.exp(torch.arange(0, hidden_dim, 2) *
-(math.log(10000.0) / hidden_dim))
pe = torch.zeros(max_len, hidden_dim)
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0).transpose(0, 1)
self.register_buffer('pe', pe)
def forward(self, x):
x = x * math.sqrt(self.hidden_dim)
x = x + self.pe[:x.size(0), :]
return x
class Encoder(nn.Module):
def __init__(self, hidden_dim, num_heads, num_layers):
super(Encoder, self).__init__()
self.layers = nn.ModuleList([EncoderLayer(hidden_dim, num_heads) for
_ in range(num_layers)])
self.norm = nn.LayerNorm(hidden_dim)
def forward(self, x):
for layer in self.layers:
x = layer(x)
x = self.norm(x)
return x
class EncoderLayer(nn.Module):
def __init__(self, hidden_dim, num_heads, dropout=0.1):
super(EncoderLayer, self).__init__()
self.self_attention = MultiHeadAttention(hidden_dim, num_heads)
self.feed_forward = FeedForward(hidden_dim)
self.norm1 = nn.LayerNorm(hidden_dim)
self.norm2 = nn.LayerNorm(hidden_dim)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
residual = x
x = self.norm1(x + self.dropout(self.self_attention(x)))
x = self.norm2(x + self.dropout(self.feed_forward(x)))
return x
class MultiHeadAttention(nn.Module):
def __init__(self, hidden_dim, num_heads):
super(MultiHeadAttention, self).__init__()
self.hidden_dim = hidden_dim
self.num_heads = num_heads
self.head_dim = hidden_dim // num_heads
self.fc_query = nn.Linear(hidden_dim, hidden_dim)
self.fc_key = nn.Linear(hidden_dim, hidden_dim)
self.fc_value = nn.Linear(hidden_dim, hidden_dim)
self.fc_out = nn.Linear(hidden_dim, hidden_dim)
def forward(self, x):
batch_size = x.size(0)
query = self.fc_query(x)
key = self.fc_key(x)
value = self.fc_value(x)
query = query.view(batch_size, -1, self.num_heads,
self.head_dim).transpose(1, 2)
key = key.view(batch_size, -1, self.num_heads,
self.head_dim).transpose(1, 2)
value = value.view(batch_size, -1, self.num_heads,
self.head_dim).transpose(1, 2)
scores = torch.matmul(query, key.transpose(-2, -1)) /
math.sqrt(self.head_dim)
scores = F.softmax(scores, dim=-1)
attended = torch.matmul(scores, value)
attended = attended.transpose(1, 2).contiguous().view(batch_size, -1,
self.hidden_dim)
output = self.fc_out(attended)
return output
class FeedForward(nn.Module):
def __init__(self, hidden_dim, dropout=0.1):
super(FeedForward, self).__init__()
self.fc1 = nn.Linear(hidden_dim, hidden_dim * 4)
self.fc2 = nn.Linear(hidden_dim * 4, hidden_dim)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
x = F.relu(self.fc1(x))
x = self.dropout(x)
x = self.fc2(x)
return x
這就是一個簡單的Transformer模型的實現範例。你可以根據自己的需求進一步擴展和調
整模型的結構和參數。
請注意，以上範例僅實現了Transformer模型的一個簡化版本，其中只包含了Encoder部分
。如果你需要實現完整的Transformer模型（包括Decoder部分），你需要進一步擴展這個
範例。
此外，還需要將你的輸入數據傳遞給這個Transformer模型，以獲得預測結果。這可以通
過創建一個Transformer對象並呼叫其forward方法來實現。例如：
model = Transformer(input_dim, hidden_dim, num_heads, num_layers)
input_data = torch.tensor([...]) # 根據你的輸入數據進行調整
output = model(input_data)
這樣你就可以使用PyTorch來建構和訓練Transformer模型了。請注意，這個範例僅提供了
一個基本的實現，你可能需要根據自己的任務和需求進行進一步的修改和擴展。
恩嗯嗯