|
@@ -156,7 +156,7 @@ class BloomModel(BloomPreTrainedModel):
|
|
|
self.n_head = config.n_head
|
|
|
|
|
|
# Embedding + LN Embedding
|
|
|
- self.word_embeddings = nn.Embedding(config.vocab_size, self.embed_dim)
|
|
|
+ self.word_embeddings = nn.Embedding(config.vocab_size, self.embed_dim)
|
|
|
self.word_embeddings_layernorm = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
|
|
|
|
|
|
# Transformer blocks
|