End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.800315643180511e-06,
-    "train_runtime": 9578.3474,
+    "train_loss": 4.212603550968197e-06,
+    "train_runtime": 8257.0992,
     "train_samples": 160,
-    "train_samples_per_second": 0.033,
+    "train_samples_per_second": 0.039,
     "train_steps_per_second": 0.002
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed211417ab4dbf1795875c45bae1617ddd077cb4123b61ae299f40001d8d81db
+oid sha256:988de3060838b6695e7f49c8a6c247cbefab7621d68be94773ceedde8975201b
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.800315643180511e-06,
-    "train_runtime": 9578.3474,
+    "train_loss": 4.212603550968197e-06,
+    "train_runtime": 8257.0992,
     "train_samples": 160,
-    "train_samples_per_second": 0.033,
+    "train_samples_per_second": 0.039,
     "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -10,192 +10,192 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 179.09375,
+      "completion_length": 191.0625,
       "epoch": 0.1,
-      "grad_norm": 3.868011713027954,
+      "grad_norm": 3.427809715270996,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 6.1962858736515045,
-      "reward_std": 1.0561962897190824,
-      "rewards/concensus_correctness_reward_func": 1.9908750094473362,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 6.149390980601311,
+      "reward_std": 0.9617523243650794,
+      "rewards/concensus_correctness_reward_func": 1.9633750040084124,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7501922326628119,
+      "rewards/question_recreation_reward_func": 0.8284847401082516,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.1114687472581863,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.1231562495231628,
       "step": 2
     },
     {
-      "completion_length": 138.28125,
+      "completion_length": 144.40625,
       "epoch": 0.2,
-      "grad_norm": 6.250236988067627,
-      "kl": 0.0048106197091328795,
+      "grad_norm": 1.3986542224884033,
+      "kl": 0.0028471651924064645,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 5.70088000735268,
-      "reward_std": 0.9463271454587812,
-      "rewards/concensus_correctness_reward_func": 1.68462498113513,
-      "rewards/consensus_reward_func": 1.625,
+      "reward": 6.339791453909129,
+      "reward_std": 0.18218231783248484,
+      "rewards/concensus_correctness_reward_func": 1.9240000024437904,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7975674602203071,
+      "rewards/question_recreation_reward_func": 0.8615414439700544,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1093125008046627,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.1636249999864958,
       "step": 4
     },
     {
-      "completion_length": 218.0625,
+      "completion_length": 169.34375,
       "epoch": 0.3,
-      "grad_norm": 4.2201995849609375,
-      "kl": 0.007397842315185699,
+      "grad_norm": 2.055198907852173,
+      "kl": 0.0038381772428692784,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 4.8540039719082415,
-      "reward_std": 1.00810604786966,
-      "rewards/concensus_correctness_reward_func": 1.3198749832808971,
-      "rewards/consensus_reward_func": 1.375,
+      "reward": 6.359603498131037,
+      "reward_std": 0.4606161545962095,
+      "rewards/concensus_correctness_reward_func": 1.9266874827444553,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6744413864798844,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9010097272694111,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.0159375043585896,
+      "rewards/strict_format_reward_func": 0.421875,
+      "rewards/xmlcount_reward_func": 1.0475312508642673,
       "step": 6
     },
     {
-      "completion_length": 145.3125,
+      "completion_length": 145.0,
       "epoch": 0.4,
-      "grad_norm": 3.8883347511291504,
-      "kl": 0.009132284259976586,
+      "grad_norm": 6.418789863586426,
+      "kl": 0.00606505721952999,
       "learning_rate": 3.5042385616324236e-07,
       "loss": -0.0,
-      "reward": 5.6693243039771914,
-      "reward_std": 0.6462988893690635,
-      "rewards/concensus_correctness_reward_func": 1.6794999912381172,
-      "rewards/consensus_reward_func": 1.75,
+      "reward": 6.4850927158258855,
+      "reward_std": 0.38101454461866524,
+      "rewards/concensus_correctness_reward_func": 1.9895624965429306,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6710430758539587,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.9008114873431623,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1312812501564622,
+      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/xmlcount_reward_func": 1.1415937505662441,
       "step": 8
     },
     {
-      "completion_length": 174.21875,
+      "completion_length": 124.4375,
       "epoch": 0.5,
-      "grad_norm": 3.7120721340179443,
-      "kl": 0.011685753270285204,
+      "grad_norm": 1.2428699731826782,
+      "kl": 0.007682406796448049,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 6.625959545373917,
-      "reward_std": 0.27632009610533714,
-      "rewards/concensus_correctness_reward_func": 2.044874995946884,
+      "reward": 6.879522025585175,
+      "reward_std": 0.05636064155260101,
+      "rewards/concensus_correctness_reward_func": 2.044374980032444,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8232720587402582,
+      "rewards/question_recreation_reward_func": 0.9757720045745373,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.2265625,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 10
     },
     {
-      "completion_length": 132.875,
+      "completion_length": 131.8125,
       "epoch": 0.6,
-      "grad_norm": 3.1394920349121094,
-      "kl": 0.014822920929873362,
+      "grad_norm": 2.1788909435272217,
+      "kl": 0.008254479664174141,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 5.989435619674623,
-      "reward_std": 0.31370167725981446,
-      "rewards/concensus_correctness_reward_func": 1.7398749850690365,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 6.214351900853217,
+      "reward_std": 0.0686783246928826,
+      "rewards/concensus_correctness_reward_func": 1.8044999837875366,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8472168361768126,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9098519282415509,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.16796875,
+      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/xmlcount_reward_func": 1.171875,
       "step": 12
     },
     {
-      "completion_length": 141.15625,
+      "completion_length": 147.125,
       "epoch": 0.7,
-      "grad_norm": 2.9059255123138428,
-      "kl": 0.010254818116663955,
+      "grad_norm": 4.228316783905029,
+      "kl": 0.012374317142530344,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 6.459207977401093,
-      "reward_std": 0.30277049803407863,
-      "rewards/concensus_correctness_reward_func": 1.98887500166893,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 6.2510038716718554,
+      "reward_std": 0.7209455195406917,
+      "rewards/concensus_correctness_reward_func": 1.8063124939799309,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8813017338979989,
+      "rewards/question_recreation_reward_func": 0.9212539186701179,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1671562492847443,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.1484375,
       "step": 14
     },
     {
-      "completion_length": 155.90625,
+      "completion_length": 156.15625,
       "epoch": 0.8,
-      "grad_norm": 5.221802234649658,
-      "kl": 0.010027846648881678,
+      "grad_norm": 2.621342658996582,
+      "kl": 0.006578712871487369,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 6.077891068533063,
-      "reward_std": 0.284588649723446,
-      "rewards/concensus_correctness_reward_func": 1.867124978452921,
+      "reward": 5.729370576096699,
+      "reward_std": 0.06670628293068148,
+      "rewards/concensus_correctness_reward_func": 1.6164999939501286,
       "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.768609810154885,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.8785581185948104,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.0827812500065193,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.078062504529953,
       "step": 16
     },
     {
-      "completion_length": 154.28125,
+      "completion_length": 116.625,
       "epoch": 0.9,
-      "grad_norm": 4.91761589050293,
-      "kl": 0.014627904194639996,
+      "grad_norm": 2.0648701190948486,
+      "kl": 0.009602756139429403,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 6.414586443454027,
-      "reward_std": 0.47418900846969336,
-      "rewards/concensus_correctness_reward_func": 1.991187497973442,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 6.675211186753586,
+      "reward_std": 0.02233249889104627,
+      "rewards/concensus_correctness_reward_func": 1.987125001847744,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.833555237040855,
+      "rewards/question_recreation_reward_func": 0.9380862333346158,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.15234375,
+      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/xmlcount_reward_func": 1.171875,
       "step": 18
     },
     {
-      "completion_length": 163.40625,
+      "completion_length": 145.09375,
       "epoch": 1.0,
-      "grad_norm": 1.2134135961532593,
-      "kl": 0.009301176614826545,
+      "grad_norm": 3.3411307334899902,
+      "kl": 0.008117947407299653,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 5.998984928242862,
-      "reward_std": 0.06303227394528221,
-      "rewards/concensus_correctness_reward_func": 1.8033749908208847,
-      "rewards/consensus_reward_func": 1.75,
+      "reward": 6.9268957525491714,
+      "reward_std": 0.217531630769372,
+      "rewards/concensus_correctness_reward_func": 2.24337500333786,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8792037358507514,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.86711449874565,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/strict_format_reward_func": 0.40625,
       "rewards/xmlcount_reward_func": 1.09765625,
       "step": 20
     },
@@ -203,9 +203,9 @@
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.800315643180511e-06,
-      "train_runtime": 9578.3474,
-      "train_samples_per_second": 0.033,
+      "train_loss": 4.212603550968197e-06,
+      "train_runtime": 8257.0992,
+      "train_samples_per_second": 0.039,
       "train_steps_per_second": 0.002
     }
   ],