Add layer types in HF config (#28)
Browse files- Add layer types in HF config (64958acc746278755e85191225eb1eae22f681f8)
Co-authored-by: Manuel de Prada <[email protected]>
- config.json +11 -0
config.json
CHANGED
|
@@ -10,6 +10,17 @@
|
|
| 10 |
"hidden_size": 4096,
|
| 11 |
"initializer_range": 0.02,
|
| 12 |
"intermediate_size": 12288,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
"max_position_embeddings": 32768,
|
| 14 |
"model_type": "mistral",
|
| 15 |
"num_attention_heads": 32,
|
|
|
|
| 10 |
"hidden_size": 4096,
|
| 11 |
"initializer_range": 0.02,
|
| 12 |
"intermediate_size": 12288,
|
| 13 |
+
"layer_types": [
|
| 14 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention",
|
| 15 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention",
|
| 16 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention",
|
| 17 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention",
|
| 18 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention",
|
| 19 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention",
|
| 20 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention",
|
| 21 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention",
|
| 22 |
+
"full_attention", "sliding_attention", "sliding_attention", "sliding_attention"
|
| 23 |
+
],
|
| 24 |
"max_position_embeddings": 32768,
|
| 25 |
"model_type": "mistral",
|
| 26 |
"num_attention_heads": 32,
|