End of training

all_results.json

@@ -1,7 +1,7 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.0067506718769436e-05,
-    "train_runtime": 8646.8913,
+    "train_loss": 1.2933736934428453e-05,
+    "train_runtime": 8542.5206,
     "train_samples": 160,
     "train_samples_per_second": 0.037,
     "train_steps_per_second": 0.002

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8edf6e37d3f187991ea4c57a5f0991da6a11e14b2c145fab540cff3ecc68f03
+oid sha256:80c3ce9613cf7cd5edabaf10e2e23f7b5a658715ce3f1a47ef84de323a45d2d8
 size 1976163472

train_results.json

@@ -1,7 +1,7 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.0067506718769436e-05,
-    "train_runtime": 8646.8913,
+    "train_loss": 1.2933736934428453e-05,
+    "train_runtime": 8542.5206,
     "train_samples": 160,
     "train_samples_per_second": 0.037,
     "train_steps_per_second": 0.002

trainer_state.json

@@ -10,201 +10,201 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 135.9375,
+      "completion_length": 138.28125,
       "epoch": 0.1,
-      "grad_norm": 4.252867698669434,
+      "grad_norm": 3.3163046836853027,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 6.640346884727478,
-      "reward_std": 0.2576066399924457,
-      "rewards/concensus_correctness_reward_func": 1.921124991029501,
+      "reward": 6.737765043973923,
+      "reward_std": 0.2534733245847747,
+      "rewards/concensus_correctness_reward_func": 1.9839374870061874,
       "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9926594123244286,
+      "rewards/question_recreation_reward_func": 0.9960150495171547,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
+      "rewards/strict_format_reward_func": 0.453125,
       "rewards/xmlcount_reward_func": 1.2421875,
       "step": 2
     },
     {
-      "completion_length": 150.75,
+      "completion_length": 118.59375,
       "epoch": 0.2,
-      "grad_norm": 1.9398584365844727,
-      "kl": 0.0011401299950648536,
+      "grad_norm": 7.513012886047363,
+      "kl": 0.020305355874597808,
       "learning_rate": 4.698684378016222e-07,
-      "loss": -0.0,
-      "reward": 6.025989001616836,
-      "reward_std": 0.6848076755850343,
-      "rewards/concensus_correctness_reward_func": 1.7485624812543392,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0,
+      "reward": 6.068916807882488,
+      "reward_std": 0.4555915691307746,
+      "rewards/concensus_correctness_reward_func": 1.8684999868273735,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.9102389980107546,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8215105133131146,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.1640625,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.08203125,
       "step": 4
     },
     {
-      "completion_length": 172.625,
+      "completion_length": 147.40625,
       "epoch": 0.3,
-      "grad_norm": 4.089184284210205,
-      "kl": 0.0072728719751467,
+      "grad_norm": 4.436612129211426,
+      "kl": 0.02294621702276345,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 6.163017577491701,
-      "reward_std": 0.07025876430270728,
-      "rewards/concensus_correctness_reward_func": 1.9294999837875366,
-      "rewards/consensus_reward_func": 1.75,
+      "reward": 6.4844411220401525,
+      "reward_std": 0.4461111004493432,
+      "rewards/concensus_correctness_reward_func": 2.1148749999701977,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8194550415500998,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.7601910710800439,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.0546875,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.078125,
       "step": 6
     },
     {
-      "completion_length": 208.6875,
+      "completion_length": 160.3125,
       "epoch": 0.4,
-      "grad_norm": 2.458192825317383,
-      "kl": 0.013654754061917629,
+      "grad_norm": 7.974029541015625,
+      "kl": 0.030128940654321923,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 6.280120316892862,
-      "reward_std": 0.17612691316753626,
-      "rewards/concensus_correctness_reward_func": 1.9237499982118607,
-      "rewards/consensus_reward_func": 1.875,
+      "reward": 5.5365669841412455,
+      "reward_std": 1.0018107543728547,
+      "rewards/concensus_correctness_reward_func": 1.8039999827742577,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9344952739775181,
+      "rewards/question_recreation_reward_func": 0.6814419475849718,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.078125,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 0.8792500011622906,
       "step": 8
     },
     {
-      "completion_length": 128.6875,
+      "completion_length": 134.15625,
       "epoch": 0.5,
-      "grad_norm": 2.5763816833496094,
-      "kl": 0.007988140281668166,
+      "grad_norm": 4.91945743560791,
+      "kl": 0.007222857192573429,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 7.199824213981628,
-      "reward_std": 0.30960790743120015,
-      "rewards/concensus_correctness_reward_func": 2.171249993145466,
+      "reward": 7.120726376771927,
+      "reward_std": 0.4210117571055889,
+      "rewards/concensus_correctness_reward_func": 2.23412499204278,
       "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9973241612315178,
+      "rewards/question_recreation_reward_func": 0.9451950751245022,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.19140625,
       "step": 10
     },
     {
-      "completion_length": 151.375,
+      "completion_length": 143.03125,
       "epoch": 0.6,
-      "grad_norm": 2.865464448928833,
-      "kl": 0.01100629365646455,
+      "grad_norm": 6.664637565612793,
+      "kl": 0.009682465242804028,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 6.153868338558823,
-      "reward_std": 0.5049338727258146,
-      "rewards/concensus_correctness_reward_func": 1.8041249811649323,
-      "rewards/consensus_reward_func": 1.75,
+      "reward": 6.475049124099314,
+      "reward_std": 0.0758700438964297,
+      "rewards/concensus_correctness_reward_func": 1.9238749742507935,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9200558536686003,
+      "rewards/question_recreation_reward_func": 0.9370491153094918,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.1328125,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.1766250003129244,
       "step": 12
     },
     {
-      "completion_length": 164.6875,
+      "completion_length": 139.4375,
       "epoch": 0.7,
-      "grad_norm": 2.9270436763763428,
-      "kl": 0.012188744651211891,
+      "grad_norm": 22.452245712280273,
+      "kl": 0.018864934208977502,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 6.698431720724329,
-      "reward_std": 0.43539159740612376,
-      "rewards/concensus_correctness_reward_func": 2.113812495023012,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 6.535804134327918,
+      "reward_std": 0.13692408660426736,
+      "rewards/concensus_correctness_reward_func": 1.9229999780654907,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8775878974702209,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.9096790780313313,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.12890625,
+      "rewards/strict_format_reward_func": 0.46875,
+      "rewards/xmlcount_reward_func": 1.171875,
       "step": 14
     },
     {
-      "completion_length": 147.875,
+      "completion_length": 173.03125,
       "epoch": 0.8,
-      "grad_norm": 4.520843029022217,
-      "kl": 0.006972079871047754,
+      "grad_norm": 15.143121719360352,
+      "kl": 0.006763902667444199,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 5.9629114486742765,
-      "reward_std": 0.09626558030140586,
-      "rewards/concensus_correctness_reward_func": 1.740624986588955,
+      "reward": 6.056088483193889,
+      "reward_std": 0.04765429539838806,
+      "rewards/concensus_correctness_reward_func": 1.8033749759197235,
       "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8472864248324186,
+      "rewards/question_recreation_reward_func": 0.8777134439442307,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.40625,
       "rewards/xmlcount_reward_func": 1.09375,
       "step": 16
     },
     {
-      "completion_length": 159.5625,
+      "completion_length": 150.21875,
       "epoch": 0.9,
-      "grad_norm": 0.99664306640625,
-      "kl": 0.014016575267305598,
+      "grad_norm": 3.726729393005371,
+      "kl": 0.0060048561899748165,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 6.428886480338406,
-      "reward_std": 0.09960044454783201,
-      "rewards/concensus_correctness_reward_func": 1.9241249784827232,
-      "rewards/consensus_reward_func": 1.875,
+      "reward": 6.480004261422437,
+      "reward_std": 0.3856835885308101,
+      "rewards/concensus_correctness_reward_func": 1.9893749840557575,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9110739563475363,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8812542363884859,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.156187504529953,
+      "rewards/strict_format_reward_func": 0.421875,
+      "rewards/xmlcount_reward_func": 1.125,
       "step": 18
     },
     {
-      "completion_length": 166.375,
+      "completion_length": 180.09375,
       "epoch": 1.0,
-      "grad_norm": 1.1509373188018799,
-      "kl": 0.011316912030451931,
+      "grad_norm": 1.9871324300765991,
+      "kl": 0.006078765269194264,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 6.039341998286545,
-      "reward_std": 0.07440098613733426,
-      "rewards/concensus_correctness_reward_func": 1.8028749898076057,
+      "reward": 6.5666842963546515,
+      "reward_std": 0.11683473509037867,
+      "rewards/concensus_correctness_reward_func": 2.05274997651577,
       "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8823419869877398,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.9045592825859785,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.0572500005364418,
+      "rewards/strict_format_reward_func": 0.375,
+      "rewards/xmlcount_reward_func": 1.109375,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.0067506718769436e-05,
-      "train_runtime": 8646.8913,
+      "train_loss": 1.2933736934428453e-05,
+      "train_runtime": 8542.5206,
       "train_samples_per_second": 0.037,
       "train_steps_per_second": 0.002
     }