Upload RetNetForCausalLM

README.md

@@ -8,4 +8,4 @@ license: mit
 datasets:
 - cerebras/SlimPajama-627B
 library_name: fla
----
+---

config.json

@@ -3,11 +3,15 @@
   "architectures": [
     "RetNetForCausalLM"
   ],
+  "attn": null,
   "attn_mode": "chunk",
   "bos_token_id": 1,
+  "conv_size": 4,
+  "elementwise_affine": true,
   "eos_token_id": 2,
   "expand_k": 1,
   "expand_v": 2,
+  "feature_map": null,
   "fuse_cross_entropy": true,
   "fuse_norm": true,
   "hidden_act": "swish",
@@ -17,12 +21,16 @@
   "intermediate_size": null,
   "max_position_embeddings": 2048,
   "model_type": "retnet",
+  "norm_eps": 1e-06,
   "num_heads": 10,
   "num_hidden_layers": 32,
+  "num_kv_heads": null,
   "rms_norm_eps": 1e-06,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.0",
+  "transformers_version": "4.48.2",
   "use_cache": true,
+  "use_output_gate": true,
+  "use_short_conv": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,6 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.40.0",
-  "use_cache": false
+  "transformers_version": "4.48.2"
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f831aa515d65e696c04cfca5da9340acce7a060742ca10a29b01533db51d8a2a
-size 4987666624
+oid sha256:853459d8b6843b6618db908d2756f1bb2e622b8e039916a84fb890422505d111
+size 4987669992

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45500f027a34a15f4f0ed940fd3d3a72513ec0bf72460f7d63cf116b627766f9
-size 457465904
+oid sha256:d3526856f41bda10650e99253396ed97aadc557b1f4d4fb9eef6eeef710ac941
+size 457466128

model.safetensors.index.json

@@ -14,6 +14,7 @@
     "model.layers.0.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -24,6 +25,7 @@
     "model.layers.1.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -34,6 +36,7 @@
     "model.layers.10.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -44,6 +47,7 @@
     "model.layers.11.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -54,6 +58,7 @@
     "model.layers.12.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -64,6 +69,7 @@
     "model.layers.13.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -74,6 +80,7 @@
     "model.layers.14.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -84,6 +91,7 @@
     "model.layers.15.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -94,6 +102,7 @@
     "model.layers.16.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -104,6 +113,7 @@
     "model.layers.17.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -114,6 +124,7 @@
     "model.layers.18.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -124,6 +135,7 @@
     "model.layers.19.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -134,6 +146,7 @@
     "model.layers.2.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -144,6 +157,7 @@
     "model.layers.20.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -154,6 +168,7 @@
     "model.layers.21.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -164,6 +179,7 @@
     "model.layers.22.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -174,6 +190,7 @@
     "model.layers.23.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -184,6 +201,7 @@
     "model.layers.24.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -194,6 +212,7 @@
     "model.layers.25.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.25.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -204,6 +223,7 @@
     "model.layers.26.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.26.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -214,6 +234,7 @@
     "model.layers.27.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.27.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -224,6 +245,7 @@
     "model.layers.28.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.28.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -234,6 +256,7 @@
     "model.layers.29.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.29.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -244,6 +267,7 @@
     "model.layers.3.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.30.attn.g_norm_swish_gate.weight": "model-00002-of-00002.safetensors",
     "model.layers.30.attn.g_proj.weight": "model-00002-of-00002.safetensors",
@@ -254,6 +278,7 @@
     "model.layers.30.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.30.mlp_norm.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.attn.g_norm_swish_gate.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.attn.g_proj.weight": "model-00002-of-00002.safetensors",
@@ -264,6 +289,7 @@
     "model.layers.31.attn_norm.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.31.mlp_norm.weight": "model-00002-of-00002.safetensors",
     "model.layers.4.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -274,6 +300,7 @@
     "model.layers.4.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -284,6 +311,7 @@
     "model.layers.5.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -294,6 +322,7 @@
     "model.layers.6.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -304,6 +333,7 @@
     "model.layers.7.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -314,6 +344,7 @@
     "model.layers.8.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.attn.g_norm_swish_gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.attn.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -324,6 +355,7 @@
     "model.layers.9.attn_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp_norm.weight": "model-00001-of-00002.safetensors",
     "model.norm.weight": "model-00002-of-00002.safetensors"
   }