Upload model + init tptt code

lora_delta_product_m0.5_constant/README.md

@@ -75,17 +75,17 @@ print(tokenizer.decode(outputs, skip_special_tokens=True))
 - **Batch size:** 1
 - **Epochs:** 1.0
 - **Learning rate (final):** N/A
-- **Loss (final):** 1.2633397308452659
-- **Training runtime:** 11512.6247 sec
-- **Samples per second:** 0.174
-- **Steps per second:** 0.174
+- **Loss (final):** 1.2270214224887408
+- **Training runtime:** 11919.5421 sec
+- **Samples per second:** 0.168
+- **Steps per second:** 0.168
 - **Total FLOPs:** 5574366965268480.0
 - **Gradient norm (final):** N/A
 
 ## Evaluation
 
 - **Metrics:** Training loss only (no eval yet, table soon : PiQA, ARC, Hella, Wino, GSM8K, MMLU)
-- **Results:** Final training loss: 1.2633397308452659
+- **Results:** Final training loss: 1.2270214224887408
 
 
 ## Citation & Contact

lora_delta_product_m0.5_constant/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa460de2d5833515a362ba04953808e73f83cd2f909de119d2605e18b79d8ec9
+oid sha256:2c2081ab470b794ec92ac84ee7460cf5e9717d100c98b572257d5616a5daa2ec
 size 27298792

lora_delta_product_m0.5_constant/config.json

@@ -43,10 +43,10 @@
     "rank_pattern": {},
     "revision": null,
     "target_modules": [
-      "q_proj",
-      "k_proj",
       "o_proj",
-      "v_proj"
+      "k_proj",
+      "v_proj",
+      "q_proj"
     ],
     "task_type": "CAUSAL_LM",
     "use_dora": false,
@@ -79,7 +79,7 @@
     "attention"
   ],
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.49.0",
+  "transformers_version": "4.51.3",
   "use_cache": true,
   "use_linear_checkpoint": true,
   "vocab_size": 32768

lora_delta_product_m0.5_constant/modeling_tptt.py

@@ -312,6 +312,7 @@ class LiZAttention(nn.Module):
             self.head_dim,
             self.num_key_value_heads,
             self.num_key_value_groups,
+            self.hidden_dim,
         ) = self._get_attention_parameters(base_attn, base_config)
         self.scaling = self.head_dim**-0.5
 
@@ -321,7 +322,7 @@ class LiZAttention(nn.Module):
             operator_mode=operator_mode,
             use_linear_checkpoint=use_linear_checkpoint,
             recurrent_config=recurrent_config,
-            hidden_dim=base_config.hidden_size,
+            hidden_dim=self.hidden_dim,
             num_heads=self.num_heads,
             head_dim=self.head_dim,
             num_key_value_heads=self.num_key_value_heads,
@@ -364,11 +365,13 @@ class LiZAttention(nn.Module):
         num_key_value_groups = getattr(base_attn, "num_key_value_groups", None) or (
             num_heads // num_key_value_heads if num_heads and num_key_value_heads else 1
         )
+        hidden_dim = getattr(base_config, "hidden_size", None) or head_dim * num_heads
         return (
             num_heads,
             head_dim,
             num_key_value_heads,
             num_key_value_groups,
+            hidden_dim,
         )
 
     def _apply_shared_projections(

lora_delta_product_m0.5_constant/runs/Aug29_07-27-47_c47f5a3d6521/events.out.tfevents.1756452484.c47f5a3d6521.19.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc53b55271fd13e60534334a32a02a17d8dba6167a60bd2d77142740b1d7fc18
+size 115789

modeling_tptt.py

@@ -312,6 +312,7 @@ class LiZAttention(nn.Module):
             self.head_dim,
             self.num_key_value_heads,
             self.num_key_value_groups,
+            self.hidden_dim,
         ) = self._get_attention_parameters(base_attn, base_config)
         self.scaling = self.head_dim**-0.5
 
@@ -321,7 +322,7 @@ class LiZAttention(nn.Module):
             operator_mode=operator_mode,
             use_linear_checkpoint=use_linear_checkpoint,
             recurrent_config=recurrent_config,
-            hidden_dim=base_config.hidden_size,
+            hidden_dim=self.hidden_dim,
             num_heads=self.num_heads,
             head_dim=self.head_dim,
             num_key_value_heads=self.num_key_value_heads,
@@ -364,11 +365,13 @@ class LiZAttention(nn.Module):
         num_key_value_groups = getattr(base_attn, "num_key_value_groups", None) or (
             num_heads // num_key_value_heads if num_heads and num_key_value_heads else 1
         )
+        hidden_dim = getattr(base_config, "hidden_size", None) or head_dim * num_heads
         return (
             num_heads,
             head_dim,
             num_key_value_heads,
             num_key_value_groups,
+            hidden_dim,
         )
 
     def _apply_shared_projections(

train_tptt.py

@@ -115,7 +115,7 @@ class LiZACallback(TrainerCallback):
         if mag_weight is not None and logs is not None:
             logs["mag_weight"] = float(mag_weight)
         if disable_linear_attn is not None and logs is not None:
-            logs["disable_linear_attn"] = not bool(disable_linear_attn)
+            logs["disable_linear_attn"] = bool(disable_linear_attn)
 
 
 def ensure_int(value: Union[int, tuple, list]) -> int: