End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.5403124709555414e-05,
-    "train_runtime": 8907.9874,
-    "train_samples": 210,
-    "train_samples_per_second": 0.036,
-    "train_steps_per_second": 0.002
+    "train_loss": 8.742677027839819e-06,
+    "train_runtime": 7708.1016,
+    "train_samples": 160,
+    "train_samples_per_second": 0.042,
+    "train_steps_per_second": 0.003
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e2b1028e5ba4d3dab0072f5eef4beff418c373894c4439372cafd818d5ac6f0
+oid sha256:e41fba43afb8c4684236b7f1df1bd3568101cab6904886e9e895f74c03da9812
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.5403124709555414e-05,
-    "train_runtime": 8907.9874,
-    "train_samples": 210,
-    "train_samples_per_second": 0.036,
-    "train_steps_per_second": 0.002
+    "train_loss": 8.742677027839819e-06,
+    "train_runtime": 7708.1016,
+    "train_samples": 160,
+    "train_samples_per_second": 0.042,
+    "train_steps_per_second": 0.003
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.7619047619047619,
+  "epoch": 1.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 148.84375,
-      "epoch": 0.0761904761904762,
-      "grad_norm": 68.10403442382812,
+      "completion_length": 158.59375,
+      "epoch": 0.1,
+      "grad_norm": 10.994775772094727,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": -0.0,
-      "reward": 0.10526368767023087,
-      "reward_std": 0.12028272116640437,
+      "loss": 0.0,
+      "reward": 0.15987260814290494,
+      "reward_std": 0.1004382234714285,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.049951187102124095,
+      "rewards/question_recreation_reward_func": 0.09737260674592108,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0071875001303851604,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 2
     },
     {
-      "completion_length": 139.75,
-      "epoch": 0.1523809523809524,
-      "grad_norm": 109.33548736572266,
-      "kl": 0.010293909310576055,
+      "completion_length": 155.71875,
+      "epoch": 0.2,
+      "grad_norm": 15.824545860290527,
+      "kl": 0.0005915943892205178,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 0.1607844315294642,
-      "reward_std": 0.13128481482999632,
+      "reward": 0.197197389556095,
+      "reward_std": 0.20802540019030857,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09828442826983519,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.0656348958145827,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.006562499795109034,
       "step": 4
     },
     {
-      "completion_length": 149.21875,
-      "epoch": 0.22857142857142856,
-      "grad_norm": 29.236942291259766,
-      "kl": 0.01587911450633328,
+      "completion_length": 89.34375,
+      "epoch": 0.3,
+      "grad_norm": 9.462965965270996,
+      "kl": 0.002917701404385298,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 0.1183337451366242,
-      "reward_std": 0.10735231206490425,
+      "reward": 0.11539923714008182,
+      "reward_std": 0.037611501840729034,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.05583375002606772,
+      "rewards/question_recreation_reward_func": 0.10883673781063408,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.006562499795109034,
       "step": 6
     },
     {
-      "completion_length": 89.90625,
-      "epoch": 0.3047619047619048,
-      "grad_norm": 31.783924102783203,
-      "kl": 0.02198620821127406,
+      "completion_length": 206.03125,
+      "epoch": 0.4,
+      "grad_norm": 5.41404390335083,
+      "kl": 0.0024468189403705765,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 0.13180370116606355,
-      "reward_std": 0.10559988473414705,
-      "rewards/concensus_correctness_reward_func": 0.0521249994635582,
+      "reward": 0.17654531029984355,
+      "reward_std": 0.1492071510583628,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.07967870321590453,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.1062328047119081,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0078125,
       "step": 8
     },
     {
-      "completion_length": 97.125,
-      "epoch": 0.38095238095238093,
-      "grad_norm": 10.334883689880371,
-      "kl": 0.03067711727999267,
+      "completion_length": 145.40625,
+      "epoch": 0.5,
+      "grad_norm": 10.185800552368164,
+      "kl": 0.0034970568067365093,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 0.057175889844074845,
-      "reward_std": 0.012211590730885291,
+      "reward": 0.10095155995804816,
+      "reward_std": 0.02929924816635321,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.057175889844074845,
+      "rewards/question_recreation_reward_func": 0.10095155995804816,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.0,
       "step": 10
     },
     {
-      "completion_length": 100.5,
-      "epoch": 0.45714285714285713,
-      "grad_norm": 19.944557189941406,
-      "kl": 0.020602296863216907,
+      "completion_length": 109.09375,
+      "epoch": 0.6,
+      "grad_norm": 23.074642181396484,
+      "kl": 0.032730046284996206,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 0.07589184356038459,
-      "reward_std": 0.02627101860343828,
+      "reward": 0.13027076213620603,
+      "reward_std": 0.04056345629214775,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.07589184356038459,
+      "rewards/question_recreation_reward_func": 0.13027076213620603,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.0,
       "step": 12
     },
     {
-      "completion_length": 116.8125,
-      "epoch": 0.5333333333333333,
-      "grad_norm": 30.17753791809082,
-      "kl": 0.017348874847812112,
+      "completion_length": 191.46875,
+      "epoch": 0.7,
+      "grad_norm": 5.876796245574951,
+      "kl": 0.005669093662618252,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 0.07637416757643223,
-      "reward_std": 0.029745125837507658,
+      "reward": 0.0862976847711252,
+      "reward_std": 0.047334227681858465,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.07637416757643223,
+      "rewards/question_recreation_reward_func": 0.10236018449359108,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.016062499955296516,
       "step": 14
     },
     {
-      "completion_length": 153.96875,
-      "epoch": 0.6095238095238096,
-      "grad_norm": 39.15425491333008,
-      "kl": 0.10780282670020824,
+      "completion_length": 80.75,
+      "epoch": 0.8,
+      "grad_norm": 13.765327453613281,
+      "kl": 0.016974578695226228,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0001,
-      "reward": 0.3936714417068288,
-      "reward_std": 0.26407188503071666,
-      "rewards/concensus_correctness_reward_func": 0.15799999982118607,
-      "rewards/consensus_reward_func": 0.125,
+      "loss": 0.0,
+      "reward": 0.210471858328674,
+      "reward_std": 0.1216582864508382,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.04817145213019103,
+      "rewards/question_recreation_reward_func": 0.1446906128549017,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.003281249897554517,
       "step": 16
     },
     {
-      "completion_length": 206.09375,
-      "epoch": 0.6857142857142857,
-      "grad_norm": 34.58672332763672,
-      "kl": 0.02152121610197355,
+      "completion_length": 92.375,
+      "epoch": 0.9,
+      "grad_norm": 14.822056770324707,
+      "kl": 0.010443305796798086,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 0.09353053948143497,
-      "reward_std": 0.028568211040692404,
+      "reward": 0.22659404878504574,
+      "reward_std": 0.18330318277003244,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09353053948143497,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.1940940455533564,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.029999999329447746,
       "step": 18
     },
     {
-      "completion_length": 226.6875,
-      "epoch": 0.7619047619047619,
-      "grad_norm": 15.182280540466309,
-      "kl": 0.008092620237221126,
+      "completion_length": 163.0625,
+      "epoch": 1.0,
+      "grad_norm": 27.081457138061523,
+      "kl": 0.012444959231743269,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.1192354466766119,
-      "reward_std": 0.0682101159618469,
-      "rewards/concensus_correctness_reward_func": 0.02162499912083149,
+      "reward": 0.0902460323413834,
+      "reward_std": 0.05573141352215316,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09761044755578041,
+      "rewards/question_recreation_reward_func": 0.0902460323413834,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
       "rewards/xmlcount_reward_func": 0.0,
       "step": 20
     },
     {
-      "epoch": 0.7619047619047619,
+      "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.5403124709555414e-05,
-      "train_runtime": 8907.9874,
-      "train_samples_per_second": 0.036,
-      "train_steps_per_second": 0.002
+      "train_loss": 8.742677027839819e-06,
+      "train_runtime": 7708.1016,
+      "train_samples_per_second": 0.042,
+      "train_steps_per_second": 0.003
     }
   ],
   "logging_steps": 2,