[Pytorch] 如何参数共享以及对参数进行相应的初始化

• 就是与embed_tokens.weight用同样的参数

if self.share_input_output_embed:
    x = F.linear(x, self.embed_tokens.weight)

• 对参数进行相应的初始化

elif not self.share_input_output_embed:
    self.embed_out = nn.Parameter(torch.Tensor(len(dictionary), output_embed_dim))
    nn.init.normal_(self.embed_out, mean=0, std=output_embed_dim ** -0.5)

if self.share_input_output_embed:
    x = F.linear(x, self.embed_tokens.weight)
else:
    x = F.linear(x, self.embed_out)

• 初始化的时候还可以让padding的部分为0

def Embedding(num_embeddings, embedding_dim, padding_idx):
    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
    nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5)
    nn.init.constant_(m.weight[padding_idx], 0)
    return m