End of training

all_results.json

@@ -1,7 +1,7 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5517725843614754e-08,
-    "train_runtime": 3164.6163,
+    "train_loss": -2.7790665626525877e-07,
+    "train_runtime": 3161.8325,
     "train_samples": 160,
     "train_samples_per_second": 0.101,
     "train_steps_per_second": 0.006

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d201826a563bcc93af96d44c916407d12947c0196732b9d5310152e84b2dc0b
+oid sha256:f56802790fcca3fcc69f9c09fbd2d62c07fab6a15110cc0d094adc7c40872245
 size 1976163472

train_results.json

@@ -1,7 +1,7 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5517725843614754e-08,
-    "train_runtime": 3164.6163,
+    "train_loss": -2.7790665626525877e-07,
+    "train_runtime": 3161.8325,
     "train_samples": 160,
     "train_samples_per_second": 0.101,
     "train_steps_per_second": 0.006

trainer_state.json

@@ -54,9 +54,9 @@
       "kl": 0.0,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 6.752281337976456,
+      "reward": 6.7521563321352005,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 2.0488750264048576,
+      "rewards/concensus_correctness_reward_func": 2.048750028014183,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
@@ -67,15 +67,15 @@
       "step": 6
     },
     {
-      "completion_length": 107.9375,
+      "completion_length": 108.1875,
       "epoch": 0.4,
       "grad_norm": 0.0,
       "kl": 0.0,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 6.62740433216095,
+      "reward": 6.627154350280762,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 2.0488750264048576,
+      "rewards/concensus_correctness_reward_func": 2.048625022172928,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
@@ -88,27 +88,27 @@
     {
       "completion_length": 107.125,
       "epoch": 0.5,
-      "grad_norm": 7.32659444224737e-08,
-      "kl": 3.538913398437593e-10,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 6.650562286376953,
-      "reward_std": 0.03314562886953354,
+      "reward": 6.673999786376953,
+      "reward_std": 0.0,
       "rewards/concensus_correctness_reward_func": 1.9240000247955322,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
       "rewards/question_recreation_reward_func": 1.0,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.484375,
-      "rewards/xmlcount_reward_func": 1.2421875,
+      "rewards/strict_format_reward_func": 0.5,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 10
     },
     {
       "completion_length": 100.4375,
       "epoch": 0.6,
-      "grad_norm": 6.133474528269289e-08,
-      "kl": 1.7213421271700113e-08,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
       "reward": 6.5019373297691345,
@@ -126,8 +126,8 @@
     {
       "completion_length": 100.8125,
       "epoch": 0.7,
-      "grad_norm": 0.003382494207471609,
-      "kl": 0.00014448053725751864,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
       "reward": 6.753007158637047,
@@ -145,8 +145,8 @@
     {
       "completion_length": 90.8125,
       "epoch": 0.8,
-      "grad_norm": 2.1626634406857193e-05,
-      "kl": 4.578997401250007e-07,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
       "reward": 6.7057811468839645,
@@ -164,8 +164,8 @@
     {
       "completion_length": 107.0,
       "epoch": 0.9,
-      "grad_norm": 1.7583732869752566e-07,
-      "kl": 7.5428810552419234e-06,
+      "grad_norm": 0.0,
+      "kl": 0.0,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
       "reward": 7.298999786376953,
@@ -183,17 +183,17 @@
     {
       "completion_length": 89.625,
       "epoch": 1.0,
-      "grad_norm": 4.646564775612205e-05,
-      "kl": 2.6783712221423173e-06,
+      "grad_norm": 3.1471806494209886e-08,
+      "kl": 0.0,
       "learning_rate": 0.0,
-      "loss": 0.0,
-      "reward": 6.455756992101669,
-      "reward_std": 0.0,
+      "loss": -0.0,
+      "reward": 6.455524355173111,
+      "reward_std": 0.00032897721393965185,
       "rewards/concensus_correctness_reward_func": 2.0490000247955322,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8755071678460808,
+      "rewards/question_recreation_reward_func": 0.8752745383681031,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.4375,
       "rewards/xmlcount_reward_func": 1.09375,
@@ -203,8 +203,8 @@
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.5517725843614754e-08,
-      "train_runtime": 3164.6163,
+      "train_loss": -2.7790665626525877e-07,
+      "train_runtime": 3161.8325,
       "train_samples_per_second": 0.101,
       "train_steps_per_second": 0.006
     }