Update configuration_bailing_moe.py

configuration_bailing_moe.py

@@ -63,7 +63,7 @@ class BailingMoeConfig(PretrainedConfig):
         self.use_sliding_window = use_sliding_window
         self.sliding_window = sliding_window
         self.max_window_layers = max_window_layers
-        self.head_dim = head_dim
+        self.head_dim = head_dim or self.hidden_size // self.num_attention_heads
         self.rope_scaling = rope_scaling
 
         # MoE configs
@@ -75,4 +75,4 @@ class BailingMoeConfig(PretrainedConfig):
         self.first_k_dense_replace = first_k_dense_replace
         self.output_router_logits = output_router_logits
 
-        super().__init__(pad_token_id=pad_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)
+        super().__init__(pad_token_id=pad_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)