前言
論文：https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf
時(shí)間：2018年6月
貢獻(xiàn)：
提出了大規(guī)模數(shù)據(jù)上無(wú)監(jiān)督預(yù)訓(xùn)練然后在目標(biāo)任務(wù)上有監(jiān)督finetune的范式。
具體實(shí)現(xiàn)
當(dāng)時(shí)由于NLP領(lǐng)域不存在像圖像領(lǐng)域中ImageNet那樣百萬(wàn)級(jí)別標(biāo)注的數(shù)據(jù)（并且圖像的像素包含了比句子更豐富的信息，百萬(wàn)級(jí)別的圖像標(biāo)注數(shù)據(jù)相當(dāng)于千萬(wàn)級(jí)別的句子標(biāo)注數(shù)據(jù)），所以當(dāng)時(shí)NLP的發(fā)展比較緩慢。本文相當(dāng)于開(kāi)疆拓土采用了在大規(guī)模數(shù)據(jù)上進(jìn)行無(wú)監(jiān)督預(yù)訓(xùn)練然后再目標(biāo)任務(wù)上進(jìn)行有監(jiān)督finetune的嘗試。
最后實(shí)驗(yàn)的效果是在12個(gè)NLP任務(wù)上，9個(gè)取得了超過(guò)SOTA的效果：
模型結(jié)構(gòu)
GPT的模型結(jié)構(gòu)核心組建是transformer的decoder模塊，為什么用transformer而不用經(jīng)典的RNN或者LSTM，GRU之類(lèi)呢？因?yàn)樽髡咴谡撐闹姓f(shuō)到，相比于RNN，transformer學(xué)到的特征更加的穩(wěn)健一些，這個(gè)可能還是跟transformer里面的self attention有關(guān)，它更加的結(jié)構(gòu)化并且可以學(xué)習(xí)了token和token之間的關(guān)系，對(duì)句子的理解更加的深刻。
完整的GPT1模型結(jié)構(gòu)也比較簡(jiǎn)單：
整體采用了12個(gè)transformer的decoder模塊構(gòu)成，其實(shí)這里說(shuō)的decoder給我造成了很多誤解，我記得transformer的decoder部分長(zhǎng)這樣：
但是看GPT論文的結(jié)構(gòu)又是transformer的encoder的樣子：
所以一直沒(méi)明白為啥說(shuō)用的是decoder，仔細(xì)看了下別人實(shí)現(xiàn)的代碼才發(fā)現(xiàn)了，主要是GPT僅僅用了單向的transformer，也就是mask multi head self attention，也就是transformer的decoder模塊的這部分：
但是整個(gè)結(jié)構(gòu)還是還是encoder一樣的，只是MHA這個(gè)地方用了mask所以說(shuō)成了用了decoder部分。
至于為什么用帶Mask的MHA呢？
GPT中因?yàn)橐瓿烧Z(yǔ)言模型的訓(xùn)練，也就要求Pre-Training預(yù)測(cè)下一個(gè)詞的時(shí)候只能夠看見(jiàn)當(dāng)前以及之前的詞，這也是GPT放棄原本Transformer的雙向結(jié)構(gòu)轉(zhuǎn)而采用單向結(jié)構(gòu)的原因。
代碼
沒(méi)有代碼是不完整的，直接上模型結(jié)構(gòu)的代碼文章來(lái)源：http://www.zghlxwxcb.cn/news/detail-503115.html
import torch
import torch.nn as nn
class ScaledDotProductAttention(nn.Module):
def __init__(self, d_k, attn_pdrop):
super(ScaledDotProductAttention, self).__init__()
self.d_k = d_k
self.dropout = nn.Dropout(attn_pdrop)
def forward(self, q, k, v, attn_mask):
# |q| : (batch_size, n_heads, q_len, d_k)
# |k| : (batch_size, n_heads, k_len, d_k)
# |v| : (batch_size, n_heads, v_len, d_v)
# |attn_mask| : (batch_size, n_heads, q_len, k_len)
attn_score = torch.matmul(q, k.transpose(-1, -2)) / (self.d_k ** 0.5)
attn_score.masked_fill_(attn_mask, -1e9)
# |attn_scroe| : (batch_size, n_heads, q_len, k_len)
attn_weights = nn.Softmax(dim=-1)(attn_score)
attn_weights = self.dropout(attn_weights)
# |attn_weights| : (batch_size, n_heads, q_len, k_len)
output = torch.matmul(attn_weights, v)
# |output| : (batch_size, n_heads, q_len, d_v)
return output, attn_weights
class MultiHeadAttention(nn.Module):
def __init__(self, d_model, n_heads, attn_pdrop):
super(MultiHeadAttention, self).__init__()
self.n_heads = n_heads
self.d_k = self.d_v = d_model // n_heads
self.WQ = nn.Linear(d_model, d_model)
self.WK = nn.Linear(d_model, d_model)
self.WV = nn.Linear(d_model, d_model)
self.scaled_dot_product_attn = ScaledDotProductAttention(self.d_k, attn_pdrop)
self.linear = nn.Linear(n_heads * self.d_v, d_model)
def forward(self, Q, K, V, attn_mask):
# |Q| : (batch_size, q_len(=seq_len), d_model)
# |K| : (batch_size, k_len(=seq_len), d_model)
# |V| : (batch_size, v_len(=seq_len), d_model)
# |attn_mask| : (batch_size, q_len, k_len)
batch_size = Q.size(0)
q_heads = self.WQ(Q).view(batch_size, -1, self.n_heads, self.d_k).transpose(1, 2)
k_heads = self.WK(K).view(batch_size, -1, self.n_heads, self.d_k).transpose(1, 2)
v_heads = self.WV(V).view(batch_size, -1, self.n_heads, self.d_v).transpose(1, 2)
# |q_heads| : (batch_size, n_heads, q_len, d_k), |k_heads| : (batch_size, n_heads, k_len, d_k), |v_heads| : (batch_size, n_heads, v_len, d_v)
attn_mask = attn_mask.unsqueeze(1).repeat(1, self.n_heads, 1, 1)
# |attn_mask| : (batch_size, n_heads, q_len, k_len)
attn, attn_weights = self.scaled_dot_product_attn(q_heads, k_heads, v_heads, attn_mask)
# |attn| : (batch_size, n_heads, q_len, d_v)
# |attn_weights| : (batch_size, n_heads, q_len, k_len)
attn = attn.transpose(1, 2).contiguous().view(batch_size, -1, self.n_heads * self.d_v)
# |attn| : (batch_size, q_len, n_heads * d_v)
outputs = self.linear(attn)
# |outputs| : (batch_size, q_len, d_model)
return outputs, attn_weights
class PositionWiseFeedForwardNetwork(nn.Module):
def __init__(self, d_model, d_ff):
super(PositionWiseFeedForwardNetwork, self).__init__()
self.linear1 = nn.Linear(d_model, d_ff)
self.linear2 = nn.Linear(d_ff, d_model)
self.gelu = nn.GELU()
nn.init.normal_(self.linear1.weight, std=0.02)
nn.init.normal_(self.linear2.weight, std=0.02)
def forward(self, inputs):
# |inputs| : (batch_size, seq_len, d_model)
outputs = self.gelu(self.linear1(inputs))
# |outputs| : (batch_size, seq_len, d_ff)
outputs = self.linear2(outputs)
# |outputs| : (batch_size, seq_len, d_model)
return outputs
class DecoderLayer(nn.Module):
def __init__(self, d_model, n_heads, d_ff, attn_pdrop, resid_pdrop):
super(DecoderLayer, self).__init__()
self.mha = MultiHeadAttention(d_model, n_heads, attn_pdrop)
self.dropout1 = nn.Dropout(resid_pdrop)
self.layernorm1 = nn.LayerNorm(d_model, eps=1e-5)
self.ffn = PositionWiseFeedForwardNetwork(d_model, d_ff)
self.dropout2 = nn.Dropout(resid_pdrop)
self.layernorm2 = nn.LayerNorm(d_model, eps=1e-5)
def forward(self, inputs, attn_mask):
# |inputs| : (batch_size, seq_len, d_model)
# |attn_mask| : (batch_size, seq_len, seq_len)
attn_outputs, attn_weights = self.mha(inputs, inputs, inputs, attn_mask)
attn_outputs = self.dropout1(attn_outputs)
attn_outputs = self.layernorm1(inputs + attn_outputs)
# |attn_outputs| : (batch_size, seq_len, d_model)
# |attn_weights| : (batch_size, n_heads, q_len(=seq_len), k_len(=seq_len))
ffn_outputs = self.ffn(attn_outputs)
ffn_outputs = self.dropout2(ffn_outputs)
ffn_outputs = self.layernorm2(attn_outputs + ffn_outputs)
# |ffn_outputs| : (batch_size, seq_len, d_model)
return ffn_outputs, attn_weights
class TransformerDecoder(nn.Module):
def __init__(self, vocab_size, seq_len, d_model, n_layers, n_heads, d_ff, embd_pdrop, attn_pdrop, resid_pdrop,
pad_id):
super(TransformerDecoder, self).__init__()
self.pad_id = pad_id
# layers
self.embedding = nn.Embedding(vocab_size, d_model)
self.dropout = nn.Dropout(embd_pdrop)
self.pos_embedding = nn.Embedding(seq_len + 1, d_model)
self.layers = nn.ModuleList(
[DecoderLayer(d_model, n_heads, d_ff, attn_pdrop, resid_pdrop) for _ in range(n_layers)])
nn.init.normal_(self.embedding.weight, std=0.02)
def forward(self, inputs):
# |inputs| : (batch_size, seq_len)
positions = torch.arange(inputs.size(1), device=inputs.device, dtype=inputs.dtype).repeat(inputs.size(0), 1) + 1
position_pad_mask = inputs.eq(self.pad_id)
positions.masked_fill_(position_pad_mask, 0)
# |positions| : (batch_size, seq_len)
outputs = self.dropout(self.embedding(inputs)) + self.pos_embedding(positions)
# |outputs| : (batch_size, seq_len, d_model)
attn_pad_mask = self.get_attention_padding_mask(inputs, inputs, self.pad_id)
# |attn_pad_mask| : (batch_size, seq_len, seq_len)
subsequent_mask = self.get_attention_subsequent_mask(inputs).to(device=attn_pad_mask.device)
# |subsequent_mask| : (batch_size, seq_len, seq_len)
attn_mask = torch.gt((attn_pad_mask.to(dtype=subsequent_mask.dtype) + subsequent_mask), 0)
# |attn_mask| : (batch_size, seq_len, seq_len)
attention_weights = []
for layer in self.layers:
outputs, attn_weights = layer(outputs, attn_mask)
# |outputs| : (batch_size, seq_len, d_model)
# |attn_weights| : (batch_size, n_heads, seq_len, seq_len)
attention_weights.append(attn_weights)
return outputs, attention_weights
def get_attention_padding_mask(self, q, k, pad_id):
attn_pad_mask = k.eq(pad_id).unsqueeze(1).repeat(1, q.size(1), 1)
# |attn_pad_mask| : (batch_size, q_len, k_len)
return attn_pad_mask
def get_attention_subsequent_mask(self, q):
bs, q_len = q.size()
subsequent_mask = torch.ones(bs, q_len, q_len).triu(diagonal=1)
# |subsequent_mask| : (batch_size, q_len, q_len)
return subsequent_mask
class GPT(nn.Module):
def __init__(self,
vocab_size,
seq_len=512,
d_model=768,
n_layers=12,
n_heads=12,
d_ff=3072,
embd_pdrop=0.1,
attn_pdrop=0.1,
resid_pdrop=0.1,
pad_id=0):
super(GPT, self).__init__()
self.decoder = TransformerDecoder(vocab_size, seq_len, d_model, n_layers, n_heads, d_ff,
embd_pdrop, attn_pdrop, resid_pdrop, pad_id)
def forward(self, inputs):
# |inputs| : (batch_size, seq_len)
outputs, attention_weights = self.decoder(inputs)
# |outputs| : (batch_size, seq_len, d_model)
# |attention_weights| : [(batch_size, n_heads, seq_len, seq_len)] * n_layers
return outputs, attention_weights
if __name__ == '__main__':
model = GPT(vocab_size=10000)
print(model)
input = torch.ones(16, 128).long()
out = model(input)
print(out[0].shape)
看著很長(zhǎng)，其實(shí)代碼很簡(jiǎn)單，就是翻譯這張圖：
首先GPT這個(gè)類(lèi)就是定義了12層transformer的decoder構(gòu)成的：
vocab_size是詞典的大小，就比如說(shuō)英文一共有10w個(gè)單詞，那么vocab_size就是10w。用來(lái)配合nn.embedding
模塊把單詞抽成embedding。
然后每個(gè)decoder里面就是對(duì)著圖寫(xiě)代碼了，其中比較核心的就是兩個(gè)mask：
文章來(lái)源地址http://www.zghlxwxcb.cn/news/detail-503115.html
- 第一個(gè)mask很好理解，因?yàn)榫渥拥拈L(zhǎng)度不一樣，比如【good morning】和【nice to meet you】，一個(gè)長(zhǎng)度是2一個(gè)是4，這樣兩個(gè)句子沒(méi)辦法組成訓(xùn)練數(shù)據(jù)，所以一般會(huì)把短的padding一下成【good morning pad pad】這樣就長(zhǎng)度一樣可以組成訓(xùn)練數(shù)據(jù)了，圖上第一個(gè)pad mask就是用于【good morning pad pad】這個(gè)里面那些是pad的部分，然后不參與self attention的計(jì)算。
- 第二個(gè)mask就是之前我困惑的地方為啥叫decoder，decoder用的是帶mask的MHA，這個(gè)mask就是圖上的第二個(gè)框，他把某個(gè)單詞后的單詞都進(jìn)行的mask。舉個(gè)例子，還是【nice to meet you】，對(duì)于nice單詞它的mask就是【0，1，1，1】，對(duì)于meet單詞，它的mask就是【0，0，0，1】，這樣從左往右相當(dāng)于去了全1矩陣的上三角的1，所以這里的mask用pytorch的triu實(shí)現(xiàn)的：
subsequent_mask = torch.ones(bs, q_len, q_len).triu(diagonal=1)
到了這里，關(guān)于Chatgpt論文筆記——GPT1詳細(xì)解讀與可運(yùn)行的代碼的文章就介紹完了。如果您還想了解更多內(nèi)容，請(qǐng)?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！
