import paddle
class EmAdd(paddle.nn.Layer):
def __init__(self, voc_size=9999, hidden_size=256):
super(EmAdd, self).__init__()
self.hidden_size = hidden_size
p = 0
while True:
voc_size //= hidden_size
if voc_size == 0:
break
else:
p += 1
self.em = paddle.nn.LayerList([paddle.nn.Embedding(hidden_size, hidden_size) for _ in range(p + 1)])
# self.em_zero = paddle.nn.LayerList([paddle.nn.Embedding(hidden_size, hidden_size,padding_idx=0)
# for _ in range(p + 1)])
def forward(self, em_add_x):
add = 0
p = len(self.em) + 1
mask = paddle.zeros(em_add_x.shape)
for i, em in enumerate(self.em):
# mask 是累加不等于0
i += 1
x0 = em_add_x % self.hidden_size ** (p - i) // self.hidden_size ** ((p - i) - 1)
文章來源地址http://www.zghlxwxcb.cn/news/detail-676077.html
文章來源：http://www.zghlxwxcb.cn/news/detail-676077.html
到了這里，關(guān)于手搓大模型之only em for pos之增加依賴性的文章就介紹完了。如果您還想了解更多內(nèi)容，請?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！
