End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.212603550968197e-06,
-    "train_runtime": 8257.0992,
+    "train_loss": 1.0067506718769436e-05,
+    "train_runtime": 8646.8913,
     "train_samples": 160,
-    "train_samples_per_second": 0.039,
+    "train_samples_per_second": 0.037,
     "train_steps_per_second": 0.002
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:988de3060838b6695e7f49c8a6c247cbefab7621d68be94773ceedde8975201b
+oid sha256:c8edf6e37d3f187991ea4c57a5f0991da6a11e14b2c145fab540cff3ecc68f03
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.212603550968197e-06,
-    "train_runtime": 8257.0992,
+    "train_loss": 1.0067506718769436e-05,
+    "train_runtime": 8646.8913,
     "train_samples": 160,
-    "train_samples_per_second": 0.039,
+    "train_samples_per_second": 0.037,
     "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 191.0625,
+      "completion_length": 135.9375,
       "epoch": 0.1,
-      "grad_norm": 3.427809715270996,
+      "grad_norm": 4.252867698669434,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 6.149390980601311,
-      "reward_std": 0.9617523243650794,
-      "rewards/concensus_correctness_reward_func": 1.9633750040084124,
-      "rewards/consensus_reward_func": 1.6875,
+      "reward": 6.640346884727478,
+      "reward_std": 0.2576066399924457,
+      "rewards/concensus_correctness_reward_func": 1.921124991029501,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8284847401082516,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9926594123244286,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1231562495231628,
+      "rewards/strict_format_reward_func": 0.421875,
+      "rewards/xmlcount_reward_func": 1.2421875,
       "step": 2
     },
     {
-      "completion_length": 144.40625,
+      "completion_length": 150.75,
       "epoch": 0.2,
-      "grad_norm": 1.3986542224884033,
-      "kl": 0.0028471651924064645,
+      "grad_norm": 1.9398584365844727,
+      "kl": 0.0011401299950648536,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0,
-      "reward": 6.339791453909129,
-      "reward_std": 0.18218231783248484,
-      "rewards/concensus_correctness_reward_func": 1.9240000024437904,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": -0.0,
+      "reward": 6.025989001616836,
+      "reward_std": 0.6848076755850343,
+      "rewards/concensus_correctness_reward_func": 1.7485624812543392,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8615414439700544,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.9102389980107546,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.1636249999864958,
+      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/xmlcount_reward_func": 1.1640625,
       "step": 4
     },
     {
-      "completion_length": 169.34375,
+      "completion_length": 172.625,
       "epoch": 0.3,
-      "grad_norm": 2.055198907852173,
-      "kl": 0.0038381772428692784,
+      "grad_norm": 4.089184284210205,
+      "kl": 0.0072728719751467,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 6.359603498131037,
-      "reward_std": 0.4606161545962095,
-      "rewards/concensus_correctness_reward_func": 1.9266874827444553,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 6.163017577491701,
+      "reward_std": 0.07025876430270728,
+      "rewards/concensus_correctness_reward_func": 1.9294999837875366,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9010097272694111,
+      "rewards/question_recreation_reward_func": 0.8194550415500998,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.0475312508642673,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.0546875,
       "step": 6
     },
     {
-      "completion_length": 145.0,
+      "completion_length": 208.6875,
       "epoch": 0.4,
-      "grad_norm": 6.418789863586426,
-      "kl": 0.00606505721952999,
+      "grad_norm": 2.458192825317383,
+      "kl": 0.013654754061917629,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": -0.0,
-      "reward": 6.4850927158258855,
-      "reward_std": 0.38101454461866524,
-      "rewards/concensus_correctness_reward_func": 1.9895624965429306,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0,
+      "reward": 6.280120316892862,
+      "reward_std": 0.17612691316753626,
+      "rewards/concensus_correctness_reward_func": 1.9237499982118607,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.9008114873431623,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9344952739775181,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.1415937505662441,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.078125,
       "step": 8
     },
     {
-      "completion_length": 124.4375,
+      "completion_length": 128.6875,
       "epoch": 0.5,
-      "grad_norm": 1.2428699731826782,
-      "kl": 0.007682406796448049,
+      "grad_norm": 2.5763816833496094,
+      "kl": 0.007988140281668166,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 6.879522025585175,
-      "reward_std": 0.05636064155260101,
-      "rewards/concensus_correctness_reward_func": 2.044374980032444,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 7.199824213981628,
+      "reward_std": 0.30960790743120015,
+      "rewards/concensus_correctness_reward_func": 2.171249993145466,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9757720045745373,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.9973241612315178,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/strict_format_reward_func": 0.46875,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 10
     },
     {
-      "completion_length": 131.8125,
+      "completion_length": 151.375,
       "epoch": 0.6,
-      "grad_norm": 2.1788909435272217,
-      "kl": 0.008254479664174141,
+      "grad_norm": 2.865464448928833,
+      "kl": 0.01100629365646455,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 6.214351900853217,
-      "reward_std": 0.0686783246928826,
-      "rewards/concensus_correctness_reward_func": 1.8044999837875366,
+      "reward": 6.153868338558823,
+      "reward_std": 0.5049338727258146,
+      "rewards/concensus_correctness_reward_func": 1.8041249811649323,
       "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9098519282415509,
+      "rewards/question_recreation_reward_func": 0.9200558536686003,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.171875,
+      "rewards/strict_format_reward_func": 0.421875,
+      "rewards/xmlcount_reward_func": 1.1328125,
       "step": 12
     },
     {
-      "completion_length": 147.125,
+      "completion_length": 164.6875,
       "epoch": 0.7,
-      "grad_norm": 4.228316783905029,
-      "kl": 0.012374317142530344,
+      "grad_norm": 2.9270436763763428,
+      "kl": 0.012188744651211891,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 6.2510038716718554,
-      "reward_std": 0.7209455195406917,
-      "rewards/concensus_correctness_reward_func": 1.8063124939799309,
-      "rewards/consensus_reward_func": 1.6875,
+      "reward": 6.698431720724329,
+      "reward_std": 0.43539159740612376,
+      "rewards/concensus_correctness_reward_func": 2.113812495023012,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9212539186701179,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.8775878974702209,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.1484375,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.12890625,
       "step": 14
     },
     {
-      "completion_length": 156.15625,
+      "completion_length": 147.875,
       "epoch": 0.8,
-      "grad_norm": 2.621342658996582,
-      "kl": 0.006578712871487369,
+      "grad_norm": 4.520843029022217,
+      "kl": 0.006972079871047754,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 5.729370576096699,
-      "reward_std": 0.06670628293068148,
-      "rewards/concensus_correctness_reward_func": 1.6164999939501286,
+      "reward": 5.9629114486742765,
+      "reward_std": 0.09626558030140586,
+      "rewards/concensus_correctness_reward_func": 1.740624986588955,
       "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8785581185948104,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8472864248324186,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.078062504529953,
+      "rewards/xmlcount_reward_func": 1.09375,
       "step": 16
     },
     {
-      "completion_length": 116.625,
+      "completion_length": 159.5625,
       "epoch": 0.9,
-      "grad_norm": 2.0648701190948486,
-      "kl": 0.009602756139429403,
+      "grad_norm": 0.99664306640625,
+      "kl": 0.014016575267305598,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 6.675211186753586,
-      "reward_std": 0.02233249889104627,
-      "rewards/concensus_correctness_reward_func": 1.987125001847744,
+      "reward": 6.428886480338406,
+      "reward_std": 0.09960044454783201,
+      "rewards/concensus_correctness_reward_func": 1.9241249784827232,
       "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9380862333346158,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9110739563475363,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.171875,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.156187504529953,
       "step": 18
     },
     {
-      "completion_length": 145.09375,
+      "completion_length": 166.375,
       "epoch": 1.0,
-      "grad_norm": 3.3411307334899902,
-      "kl": 0.008117947407299653,
+      "grad_norm": 1.1509373188018799,
+      "kl": 0.011316912030451931,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 6.9268957525491714,
-      "reward_std": 0.217531630769372,
-      "rewards/concensus_correctness_reward_func": 2.24337500333786,
-      "rewards/consensus_reward_func": 1.6875,
+      "reward": 6.039341998286545,
+      "reward_std": 0.07440098613733426,
+      "rewards/concensus_correctness_reward_func": 1.8028749898076057,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.86711449874565,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8823419869877398,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.09765625,
+      "rewards/strict_format_reward_func": 0.421875,
+      "rewards/xmlcount_reward_func": 1.0572500005364418,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 4.212603550968197e-06,
-      "train_runtime": 8257.0992,
-      "train_samples_per_second": 0.039,
+      "train_loss": 1.0067506718769436e-05,
+      "train_runtime": 8646.8913,
+      "train_samples_per_second": 0.037,
       "train_steps_per_second": 0.002
     }
   ],