hidden_act_gated = True

hivemind/moe/server/layers/albert.py

@@ -46,7 +46,7 @@ _TOKENIZER_FOR_DOC = "AlbertTokenizer"
 
 class LeanAlbertConfig(AlbertConfig):
     rotary_embedding_base: int = 10_000
-    hidden_act_gated: bool = False
+    hidden_act_gated: bool = True
 
     def __hash__(self):
         return hash("\t".join(f"{k}={v}" for k, v in self.__dict__.items() if not k.startswith("_")))