Upload folder using huggingface_hub

.gitattributes

@@ -63,3 +63,4 @@ metrics.jsonl filter=lfs diff=lfs merge=lfs -text
 nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024/hf/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 nemotron_synthetic_1T_exp_7B/metrics.jsonl filter=lfs diff=lfs merge=lfs -text
 nemotron_synthetic_1T_exp_7B/hf/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/tokenizer.json filter=lfs diff=lfs merge=lfs -text

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/README.md

@@ -0,0 +1,68 @@
+---
+library_name: transformers
+model_name: nemotron-fineinstructions-7b-dpo
+tags:
+- generated_from_trainer
+- dpo
+- trl
+licence: license
+---
+
+# Model Card for nemotron-fineinstructions-7b-dpo
+
+This model is a fine-tuned version of [None](https://huggingface.co/None).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="craffel/nemotron-fineinstructions-7b-dpo", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.25.0
+- Transformers: 4.57.1
+- Pytorch: 2.6.0+cu126
+- Datasets: 4.4.1
+- Tokenizers: 0.22.1
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -3.674851179122925,
+    "eval_logits/rejected": -3.7333829402923584,
+    "eval_logps/chosen": -321.22222900390625,
+    "eval_logps/rejected": -301.6666564941406,
+    "eval_loss": 0.6219358444213867,
+    "eval_rewards/accuracies": 0.6696428656578064,
+    "eval_rewards/chosen": 1.1342695951461792,
+    "eval_rewards/margins": 1.3358135223388672,
+    "eval_rewards/rejected": -0.2014663964509964,
+    "eval_runtime": 37.404,
+    "eval_samples_per_second": 53.47,
+    "eval_steps_per_second": 1.684,
+    "total_flos": 0.0,
+    "train_loss": 0.3160468312902597,
+    "train_runtime": 10869.2749,
+    "train_samples": 61135,
+    "train_samples_per_second": 16.874,
+    "train_steps_per_second": 0.132
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/chat_template.jinja

@@ -0,0 +1 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for msg in messages %}{% if msg.role=='user' %}{% if loop.index > 1 %}{{ '\n\n' }}{% endif %}Instruction: {{ msg.content }}{% elif msg.role=='assistant' %}{{ '\n\nAnswer:' }}{{ msg.content }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '\n\nAnswer:' }}{% endif %}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 128001,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 100000.0,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/eval_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 3.0,
+    "eval_logits/chosen": -3.674851179122925,
+    "eval_logits/rejected": -3.7333829402923584,
+    "eval_logps/chosen": -321.22222900390625,
+    "eval_logps/rejected": -301.6666564941406,
+    "eval_loss": 0.6219358444213867,
+    "eval_rewards/accuracies": 0.6696428656578064,
+    "eval_rewards/chosen": 1.1342695951461792,
+    "eval_rewards/margins": 1.3358135223388672,
+    "eval_rewards/rejected": -0.2014663964509964,
+    "eval_runtime": 37.404,
+    "eval_samples_per_second": 53.47,
+    "eval_steps_per_second": 1.684
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001
+  ],
+  "pad_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.57.1"
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:619fd5a8fdd0f4f144bc908bc16c03af4df7c75713bef7499a6f5e3054a32a93
+size 4917979000

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:450145dcb5aad5b85e871272fc6cbf99d80861b628c547d7877c413dc42f6b17
+size 4947390880

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc982ec6dd50e63f6a47b39260d4291a51aa7dc6bc31a29432ed8b16d4a45dfe
+size 4137880232

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff3f68bfadb15f87cd47e7d25b5b8d82f4680184d11573c700d44bb3ff2cd1e5
+size 1050673280

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/model.safetensors.index.json

@@ -0,0 +1,299 @@
+{
+  "metadata": {
+    "total_parameters": 266240,
+    "total_size": 15053889536
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/runs/Nov24_20-03-59_ip-26-0-165-59/events.out.tfevents.1764014736.ip-26-0-165-59.1711455.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05c16467ba012eca658992ebb92374dfb82dbe8c96cbc8f1059f3f36506c7f36
+size 6242

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/runs/Nov24_20-08-04_ip-26-0-171-102/events.out.tfevents.1764014910.ip-26-0-171-102.1183628.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a1a0bd1583d3e466c88f7873ccfc9eb1698838d01875235ceea6fa457b149f0
+size 115171

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/runs/Nov24_20-08-04_ip-26-0-171-102/events.out.tfevents.1764025817.ip-26-0-171-102.1183628.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aeffaee456e5ad712748de4654b374844107abc1bda0f2637f36272c6b541b3
+size 828

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|end_of_text|>"
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393
+size 17209961

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for msg in messages %}{% if msg.role=='user' %}{% if loop.index > 1 %}{{ '\\n\\n' }}{% endif %}Instruction: {{ msg.content }}{% elif msg.role=='assistant' %}{{ '\\n\\nAnswer:' }}{{ msg.content }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '\\n\\nAnswer:' }}{% endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 0.0,
+    "train_loss": 0.3160468312902597,
+    "train_runtime": 10869.2749,
+    "train_samples": 61135,
+    "train_samples_per_second": 16.874,
+    "train_steps_per_second": 0.132
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/trainer_state.json

@@ -0,0 +1,2412 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 100,
+  "global_step": 1434,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 67.94458938040499,
+      "learning_rate": 6.25e-08,
+      "logits/chosen": -3.7376952171325684,
+      "logits/rejected": -3.844921827316284,
+      "logps/chosen": -342.63751220703125,
+      "logps/rejected": -306.23748779296875,
+      "loss": 0.7193,
+      "rewards/accuracies": 0.13906249403953552,
+      "rewards/chosen": 0.007275390438735485,
+      "rewards/margins": 0.00537109375,
+      "rewards/rejected": 0.0019042969215661287,
+      "step": 10
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 70.00034801068554,
+      "learning_rate": 1.3194444444444444e-07,
+      "logits/chosen": -3.711718797683716,
+      "logits/rejected": -3.8062500953674316,
+      "logps/chosen": -325.75,
+      "logps/rejected": -283.4750061035156,
+      "loss": 0.7265,
+      "rewards/accuracies": 0.21640625596046448,
+      "rewards/chosen": 0.01828613318502903,
+      "rewards/margins": 0.00986328162252903,
+      "rewards/rejected": 0.0084228515625,
+      "step": 20
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 73.84061906776567,
+      "learning_rate": 2.013888888888889e-07,
+      "logits/chosen": -3.7281250953674316,
+      "logits/rejected": -3.819140672683716,
+      "logps/chosen": -320.7250061035156,
+      "logps/rejected": -290.3999938964844,
+      "loss": 0.742,
+      "rewards/accuracies": 0.19218750298023224,
+      "rewards/chosen": 0.01606445387005806,
+      "rewards/margins": -0.01552734337747097,
+      "rewards/rejected": 0.03159179538488388,
+      "step": 30
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 74.75824316824546,
+      "learning_rate": 2.708333333333333e-07,
+      "logits/chosen": -3.7691407203674316,
+      "logits/rejected": -3.8853516578674316,
+      "logps/chosen": -334.625,
+      "logps/rejected": -294.2875061035156,
+      "loss": 0.7315,
+      "rewards/accuracies": 0.21718749403953552,
+      "rewards/chosen": 0.05500488355755806,
+      "rewards/margins": 0.0233154296875,
+      "rewards/rejected": 0.03168945387005806,
+      "step": 40
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 71.62286659469315,
+      "learning_rate": 3.402777777777778e-07,
+      "logits/chosen": -3.709179639816284,
+      "logits/rejected": -3.75,
+      "logps/chosen": -339.54998779296875,
+      "logps/rejected": -308.5874938964844,
+      "loss": 0.7217,
+      "rewards/accuracies": 0.21250000596046448,
+      "rewards/chosen": 0.07255859673023224,
+      "rewards/margins": 0.03286132961511612,
+      "rewards/rejected": 0.03969726711511612,
+      "step": 50
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 69.44193644581043,
+      "learning_rate": 4.0972222222222217e-07,
+      "logits/chosen": -3.7298827171325684,
+      "logits/rejected": -3.856640577316284,
+      "logps/chosen": -352.29998779296875,
+      "logps/rejected": -291.8125,
+      "loss": 0.6966,
+      "rewards/accuracies": 0.2718749940395355,
+      "rewards/chosen": 0.144775390625,
+      "rewards/margins": 0.10195312649011612,
+      "rewards/rejected": 0.04279785230755806,
+      "step": 60
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 64.12099150848341,
+      "learning_rate": 4.791666666666667e-07,
+      "logits/chosen": -3.7173829078674316,
+      "logits/rejected": -3.814453125,
+      "logps/chosen": -330.1312561035156,
+      "logps/rejected": -296.4624938964844,
+      "loss": 0.7026,
+      "rewards/accuracies": 0.32343751192092896,
+      "rewards/chosen": 0.25859373807907104,
+      "rewards/margins": 0.14349365234375,
+      "rewards/rejected": 0.11510010063648224,
+      "step": 70
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 71.54599133028879,
+      "learning_rate": 5.486111111111111e-07,
+      "logits/chosen": -3.8140625953674316,
+      "logits/rejected": -3.945507764816284,
+      "logps/chosen": -323.8500061035156,
+      "logps/rejected": -287.875,
+      "loss": 0.6948,
+      "rewards/accuracies": 0.3734374940395355,
+      "rewards/chosen": 0.30598145723342896,
+      "rewards/margins": 0.18842773139476776,
+      "rewards/rejected": 0.1175537109375,
+      "step": 80
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 71.27315856645413,
+      "learning_rate": 6.180555555555556e-07,
+      "logits/chosen": -3.710156202316284,
+      "logits/rejected": -3.8193359375,
+      "logps/chosen": -329.2875061035156,
+      "logps/rejected": -282.04998779296875,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.40312498807907104,
+      "rewards/chosen": 0.43291014432907104,
+      "rewards/margins": 0.2662109434604645,
+      "rewards/rejected": 0.16669921576976776,
+      "step": 90
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 61.625231993468525,
+      "learning_rate": 6.875e-07,
+      "logits/chosen": -3.726757764816284,
+      "logits/rejected": -3.8509764671325684,
+      "logps/chosen": -340.5,
+      "logps/rejected": -290.95001220703125,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.45390623807907104,
+      "rewards/chosen": 0.47233885526657104,
+      "rewards/margins": 0.29780274629592896,
+      "rewards/rejected": 0.17448730766773224,
+      "step": 100
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "eval_logits/chosen": -3.7497520446777344,
+      "eval_logits/rejected": -3.8070435523986816,
+      "eval_logps/chosen": -322.5079345703125,
+      "eval_logps/rejected": -300.8888854980469,
+      "eval_loss": 0.6892461180686951,
+      "eval_rewards/accuracies": 0.4588293731212616,
+      "eval_rewards/chosen": 0.4297184944152832,
+      "eval_rewards/margins": 0.2719804048538208,
+      "eval_rewards/rejected": 0.1578620970249176,
+      "eval_runtime": 37.5579,
+      "eval_samples_per_second": 53.251,
+      "eval_steps_per_second": 1.677,
+      "step": 100
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 76.47162928607449,
+      "learning_rate": 7.569444444444444e-07,
+      "logits/chosen": -3.738085985183716,
+      "logits/rejected": -3.8189454078674316,
+      "logps/chosen": -326.3999938964844,
+      "logps/rejected": -301.88751220703125,
+      "loss": 0.6947,
+      "rewards/accuracies": 0.47578126192092896,
+      "rewards/chosen": 0.457763671875,
+      "rewards/margins": 0.2734130918979645,
+      "rewards/rejected": 0.18430176377296448,
+      "step": 110
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 67.20159639071015,
+      "learning_rate": 8.263888888888888e-07,
+      "logits/chosen": -3.744140625,
+      "logits/rejected": -3.829296827316284,
+      "logps/chosen": -339.67498779296875,
+      "logps/rejected": -299.07501220703125,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.4984374940395355,
+      "rewards/chosen": 0.5806884765625,
+      "rewards/margins": 0.35126954317092896,
+      "rewards/rejected": 0.229248046875,
+      "step": 120
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 137.25543545176455,
+      "learning_rate": 8.958333333333334e-07,
+      "logits/chosen": -3.687695264816284,
+      "logits/rejected": -3.7857422828674316,
+      "logps/chosen": -324.2124938964844,
+      "logps/rejected": -283.25,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.49296873807907104,
+      "rewards/chosen": 0.533398449420929,
+      "rewards/margins": 0.3274902403354645,
+      "rewards/rejected": 0.20590820908546448,
+      "step": 130
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 69.51075249598249,
+      "learning_rate": 9.652777777777777e-07,
+      "logits/chosen": -3.824414014816284,
+      "logits/rejected": -3.927539110183716,
+      "logps/chosen": -325.17498779296875,
+      "logps/rejected": -293.9624938964844,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.526562511920929,
+      "rewards/chosen": 0.6219482421875,
+      "rewards/margins": 0.388671875,
+      "rewards/rejected": 0.23322753608226776,
+      "step": 140
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 59.810996644761985,
+      "learning_rate": 9.999629323566322e-07,
+      "logits/chosen": -3.6976561546325684,
+      "logits/rejected": -3.8392577171325684,
+      "logps/chosen": -333.1625061035156,
+      "logps/rejected": -294.4312438964844,
+      "loss": 0.6631,
+      "rewards/accuracies": 0.535937488079071,
+      "rewards/chosen": 0.7535156011581421,
+      "rewards/margins": 0.4314331114292145,
+      "rewards/rejected": 0.3218994140625,
+      "step": 150
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 74.26157723136677,
+      "learning_rate": 9.996664241851196e-07,
+      "logits/chosen": -3.7867188453674316,
+      "logits/rejected": -3.8519530296325684,
+      "logps/chosen": -318.57501220703125,
+      "logps/rejected": -283.5874938964844,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.522656261920929,
+      "rewards/chosen": 0.7391601800918579,
+      "rewards/margins": 0.40117186307907104,
+      "rewards/rejected": 0.3381103575229645,
+      "step": 160
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 58.138124188234364,
+      "learning_rate": 9.990735836893225e-07,
+      "logits/chosen": -3.685546875,
+      "logits/rejected": -3.781054735183716,
+      "logps/chosen": -329.625,
+      "logps/rejected": -301.4125061035156,
+      "loss": 0.6185,
+      "rewards/accuracies": 0.5726562738418579,
+      "rewards/chosen": 0.8713623285293579,
+      "rewards/margins": 0.6025635004043579,
+      "rewards/rejected": 0.26860350370407104,
+      "step": 170
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 54.60851339972346,
+      "learning_rate": 9.98184762459409e-07,
+      "logits/chosen": -3.7083983421325684,
+      "logits/rejected": -3.841796875,
+      "logps/chosen": -321.2250061035156,
+      "logps/rejected": -289.9624938964844,
+      "loss": 0.6187,
+      "rewards/accuracies": 0.5765625238418579,
+      "rewards/chosen": 0.7500976324081421,
+      "rewards/margins": 0.575122058391571,
+      "rewards/rejected": 0.17526856064796448,
+      "step": 180
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 62.1465703839228,
+      "learning_rate": 9.97000487619973e-07,
+      "logits/chosen": -3.6908202171325684,
+      "logits/rejected": -3.857617139816284,
+      "logps/chosen": -326.51251220703125,
+      "logps/rejected": -287.7749938964844,
+      "loss": 0.636,
+      "rewards/accuracies": 0.5882812738418579,
+      "rewards/chosen": 0.7892090082168579,
+      "rewards/margins": 0.5947021245956421,
+      "rewards/rejected": 0.19438476860523224,
+      "step": 190
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 64.56167173201452,
+      "learning_rate": 9.955214615174171e-07,
+      "logits/chosen": -3.73828125,
+      "logits/rejected": -3.796875,
+      "logps/chosen": -335.01251220703125,
+      "logps/rejected": -303.01251220703125,
+      "loss": 0.656,
+      "rewards/accuracies": 0.5726562738418579,
+      "rewards/chosen": 0.825024425983429,
+      "rewards/margins": 0.5650879144668579,
+      "rewards/rejected": 0.26008301973342896,
+      "step": 200
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "eval_logits/chosen": -3.7338788509368896,
+      "eval_logits/rejected": -3.793154716491699,
+      "eval_logps/chosen": -321.77777099609375,
+      "eval_logps/rejected": -300.9206237792969,
+      "eval_loss": 0.6212343573570251,
+      "eval_rewards/accuracies": 0.5853174328804016,
+      "eval_rewards/chosen": 0.78125,
+      "eval_rewards/margins": 0.6164434552192688,
+      "eval_rewards/rejected": 0.1648840457201004,
+      "eval_runtime": 37.4548,
+      "eval_samples_per_second": 53.398,
+      "eval_steps_per_second": 1.682,
+      "step": 200
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 74.47483539484082,
+      "learning_rate": 9.937485613034207e-07,
+      "logits/chosen": -3.773632764816284,
+      "logits/rejected": -3.8617186546325684,
+      "logps/chosen": -330.5375061035156,
+      "logps/rejected": -280.7749938964844,
+      "loss": 0.6347,
+      "rewards/accuracies": 0.5726562738418579,
+      "rewards/chosen": 0.780957043170929,
+      "rewards/margins": 0.596118152141571,
+      "rewards/rejected": 0.18452148139476776,
+      "step": 210
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 70.66759241808225,
+      "learning_rate": 9.91682838414733e-07,
+      "logits/chosen": -3.7994141578674316,
+      "logits/rejected": -3.861132860183716,
+      "logps/chosen": -327.38751220703125,
+      "logps/rejected": -289.48748779296875,
+      "loss": 0.6432,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.8098388910293579,
+      "rewards/margins": 0.570605456829071,
+      "rewards/rejected": 0.23928222060203552,
+      "step": 220
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 62.599416258115795,
+      "learning_rate": 9.893255179496105e-07,
+      "logits/chosen": -3.7132811546325684,
+      "logits/rejected": -3.808789014816284,
+      "logps/chosen": -340.48748779296875,
+      "logps/rejected": -305.0249938964844,
+      "loss": 0.6416,
+      "rewards/accuracies": 0.571093738079071,
+      "rewards/chosen": 0.8509277105331421,
+      "rewards/margins": 0.606762707233429,
+      "rewards/rejected": 0.24433593451976776,
+      "step": 230
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 68.37176785466319,
+      "learning_rate": 9.866779979412583e-07,
+      "logits/chosen": -3.7074217796325684,
+      "logits/rejected": -3.795117139816284,
+      "logps/chosen": -331.61248779296875,
+      "logps/rejected": -285.6000061035156,
+      "loss": 0.6546,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": 0.8475586175918579,
+      "rewards/margins": 0.576367199420929,
+      "rewards/rejected": 0.2711425721645355,
+      "step": 240
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 89.6046858741352,
+      "learning_rate": 9.837418485287126e-07,
+      "logits/chosen": -3.7437500953674316,
+      "logits/rejected": -3.8167967796325684,
+      "logps/chosen": -348.3500061035156,
+      "logps/rejected": -314.5,
+      "loss": 0.6469,
+      "rewards/accuracies": 0.5804687738418579,
+      "rewards/chosen": 0.850390613079071,
+      "rewards/margins": 0.637402355670929,
+      "rewards/rejected": 0.2130126953125,
+      "step": 250
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 66.20129779816008,
+      "learning_rate": 9.805188110256532e-07,
+      "logits/chosen": -3.7017579078674316,
+      "logits/rejected": -3.814453125,
+      "logps/chosen": -313.54998779296875,
+      "logps/rejected": -292.7875061035156,
+      "loss": 0.6317,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": 0.7490234375,
+      "rewards/margins": 0.642993152141571,
+      "rewards/rejected": 0.10600586235523224,
+      "step": 260
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 59.46335465782688,
+      "learning_rate": 9.770107968877002e-07,
+      "logits/chosen": -3.6861329078674316,
+      "logits/rejected": -3.757617235183716,
+      "logps/chosen": -325.4624938964844,
+      "logps/rejected": -290.86248779296875,
+      "loss": 0.6158,
+      "rewards/accuracies": 0.600781261920929,
+      "rewards/chosen": 0.7353271245956421,
+      "rewards/margins": 0.641406238079071,
+      "rewards/rejected": 0.09392090141773224,
+      "step": 270
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 56.59501031380489,
+      "learning_rate": 9.732198865788047e-07,
+      "logits/chosen": -3.6982421875,
+      "logits/rejected": -3.819140672683716,
+      "logps/chosen": -320.4375,
+      "logps/rejected": -290.36248779296875,
+      "loss": 0.627,
+      "rewards/accuracies": 0.5960937738418579,
+      "rewards/chosen": 0.8112548589706421,
+      "rewards/margins": 0.674023449420929,
+      "rewards/rejected": 0.13718262314796448,
+      "step": 280
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 57.797512517561366,
+      "learning_rate": 9.691483283374083e-07,
+      "logits/chosen": -3.633984327316284,
+      "logits/rejected": -3.7310547828674316,
+      "logps/chosen": -326.0874938964844,
+      "logps/rejected": -292.79998779296875,
+      "loss": 0.62,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": 0.8121337890625,
+      "rewards/margins": 0.671459972858429,
+      "rewards/rejected": 0.14047852158546448,
+      "step": 290
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 65.10585842475264,
+      "learning_rate": 9.647985368431032e-07,
+      "logits/chosen": -3.695117235183716,
+      "logits/rejected": -3.809375047683716,
+      "logps/chosen": -333.48748779296875,
+      "logps/rejected": -310.6000061035156,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": 0.819262683391571,
+      "rewards/margins": 0.6747070550918579,
+      "rewards/rejected": 0.14433594048023224,
+      "step": 300
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "eval_logits/chosen": -3.7036211490631104,
+      "eval_logits/rejected": -3.7661211490631104,
+      "eval_logps/chosen": -321.96826171875,
+      "eval_logps/rejected": -301.23809814453125,
+      "eval_loss": 0.6238330006599426,
+      "eval_rewards/accuracies": 0.6076388955116272,
+      "eval_rewards/chosen": 0.7393662929534912,
+      "eval_rewards/margins": 0.7346230149269104,
+      "eval_rewards/rejected": 0.0047588045708835125,
+      "eval_runtime": 37.5625,
+      "eval_samples_per_second": 53.245,
+      "eval_steps_per_second": 1.677,
+      "step": 300
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 54.94376028011871,
+      "learning_rate": 9.601730917845796e-07,
+      "logits/chosen": -3.757031202316284,
+      "logits/rejected": -3.817187547683716,
+      "logps/chosen": -321.8999938964844,
+      "logps/rejected": -294.0874938964844,
+      "loss": 0.6415,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": 0.7624756097793579,
+      "rewards/margins": 0.6733154058456421,
+      "rewards/rejected": 0.08940429985523224,
+      "step": 310
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 54.60796628475133,
+      "learning_rate": 9.552747363297171e-07,
+      "logits/chosen": -3.6624999046325684,
+      "logits/rejected": -3.7740235328674316,
+      "logps/chosen": -324.75,
+      "logps/rejected": -282.07501220703125,
+      "loss": 0.5951,
+      "rewards/accuracies": 0.63671875,
+      "rewards/chosen": 0.798095703125,
+      "rewards/margins": 0.820263683795929,
+      "rewards/rejected": -0.02257080003619194,
+      "step": 320
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 65.19625788187524,
+      "learning_rate": 9.501063754987187e-07,
+      "logits/chosen": -3.748046875,
+      "logits/rejected": -3.8167967796325684,
+      "logps/chosen": -322.88751220703125,
+      "logps/rejected": -284.82501220703125,
+      "loss": 0.6136,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": 0.843505859375,
+      "rewards/margins": 0.7599121332168579,
+      "rewards/rejected": 0.0836181640625,
+      "step": 330
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 64.14203777677045,
+      "learning_rate": 9.446710744412594e-07,
+      "logits/chosen": -3.709765672683716,
+      "logits/rejected": -3.7955079078674316,
+      "logps/chosen": -330.7875061035156,
+      "logps/rejected": -294.4750061035156,
+      "loss": 0.6415,
+      "rewards/accuracies": 0.5992187261581421,
+      "rewards/chosen": 0.7884277105331421,
+      "rewards/margins": 0.7271484136581421,
+      "rewards/rejected": 0.06125488132238388,
+      "step": 340
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 70.39266071618265,
+      "learning_rate": 9.38972056618668e-07,
+      "logits/chosen": -3.646777391433716,
+      "logits/rejected": -3.724414110183716,
+      "logps/chosen": -338.7875061035156,
+      "logps/rejected": -298.4750061035156,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": 0.7911621332168579,
+      "rewards/margins": 0.7145751714706421,
+      "rewards/rejected": 0.07685546576976776,
+      "step": 350
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 61.588326947739354,
+      "learning_rate": 9.330127018922193e-07,
+      "logits/chosen": -3.6732420921325684,
+      "logits/rejected": -3.779296875,
+      "logps/chosen": -342.07501220703125,
+      "logps/rejected": -302.75,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.6070312261581421,
+      "rewards/chosen": 0.800732433795929,
+      "rewards/margins": 0.685791015625,
+      "rewards/rejected": 0.11477050930261612,
+      "step": 360
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 64.07501369204914,
+      "learning_rate": 9.267965445186731e-07,
+      "logits/chosen": -3.642382860183716,
+      "logits/rejected": -3.719531297683716,
+      "logps/chosen": -332.54998779296875,
+      "logps/rejected": -299.79998779296875,
+      "loss": 0.6279,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": 0.8393799066543579,
+      "rewards/margins": 0.7691650390625,
+      "rewards/rejected": 0.07014159858226776,
+      "step": 370
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 65.86507877374282,
+      "learning_rate": 9.203272710542469e-07,
+      "logits/chosen": -3.628710985183716,
+      "logits/rejected": -3.7119140625,
+      "logps/chosen": -339.5249938964844,
+      "logps/rejected": -297.61248779296875,
+      "loss": 0.6303,
+      "rewards/accuracies": 0.6117187738418579,
+      "rewards/chosen": 0.925830066204071,
+      "rewards/margins": 0.7783203125,
+      "rewards/rejected": 0.14697265625,
+      "step": 380
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 68.78575863617323,
+      "learning_rate": 9.136087181682649e-07,
+      "logits/chosen": -3.7412109375,
+      "logits/rejected": -3.8609375953674316,
+      "logps/chosen": -329.3125,
+      "logps/rejected": -289.2250061035156,
+      "loss": 0.6283,
+      "rewards/accuracies": 0.610156238079071,
+      "rewards/chosen": 0.9496825933456421,
+      "rewards/margins": 0.772216796875,
+      "rewards/rejected": 0.17724609375,
+      "step": 390
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 68.87899431742649,
+      "learning_rate": 9.066448703677827e-07,
+      "logits/chosen": -3.693164110183716,
+      "logits/rejected": -3.796093702316284,
+      "logps/chosen": -325.125,
+      "logps/rejected": -301.4125061035156,
+      "loss": 0.6368,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.8935546875,
+      "rewards/margins": 0.8002685308456421,
+      "rewards/rejected": 0.09360351413488388,
+      "step": 400
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "eval_logits/chosen": -3.7078373432159424,
+      "eval_logits/rejected": -3.7668650150299072,
+      "eval_logps/chosen": -321.5555419921875,
+      "eval_logps/rejected": -301.0952453613281,
+      "eval_loss": 0.6099804639816284,
+      "eval_rewards/accuracies": 0.6111111044883728,
+      "eval_rewards/chosen": 0.872860848903656,
+      "eval_rewards/margins": 0.811600923538208,
+      "eval_rewards/rejected": 0.0607793889939785,
+      "eval_runtime": 37.4628,
+      "eval_samples_per_second": 53.386,
+      "eval_steps_per_second": 1.682,
+      "step": 400
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 75.62350562713198,
+      "learning_rate": 8.994398576345334e-07,
+      "logits/chosen": -3.765625,
+      "logits/rejected": -3.8199219703674316,
+      "logps/chosen": -338.3125,
+      "logps/rejected": -299.70001220703125,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": 0.9367920160293579,
+      "rewards/margins": 0.824877917766571,
+      "rewards/rejected": 0.11210937798023224,
+      "step": 410
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 60.609570345392605,
+      "learning_rate": 8.919979529756007e-07,
+      "logits/chosen": -3.740429639816284,
+      "logits/rejected": -3.809375047683716,
+      "logps/chosen": -339.57501220703125,
+      "logps/rejected": -302.51251220703125,
+      "loss": 0.5986,
+      "rewards/accuracies": 0.6265624761581421,
+      "rewards/chosen": 0.9933105707168579,
+      "rewards/margins": 0.902050793170929,
+      "rewards/rejected": 0.09113769233226776,
+      "step": 420
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 63.48254376519747,
+      "learning_rate": 8.843235698892659e-07,
+      "logits/chosen": -3.7212891578674316,
+      "logits/rejected": -3.785351514816284,
+      "logps/chosen": -336.1499938964844,
+      "logps/rejected": -298.36248779296875,
+      "loss": 0.6255,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": 0.9171142578125,
+      "rewards/margins": 0.7958984375,
+      "rewards/rejected": 0.1212158203125,
+      "step": 430
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 62.62221279337957,
+      "learning_rate": 8.764212597475396e-07,
+      "logits/chosen": -3.685351610183716,
+      "logits/rejected": -3.781445264816284,
+      "logps/chosen": -349.6875,
+      "logps/rejected": -310.2250061035156,
+      "loss": 0.6286,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.870532214641571,
+      "rewards/margins": 0.850268542766571,
+      "rewards/rejected": 0.02048339881002903,
+      "step": 440
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 64.19782895102308,
+      "learning_rate": 8.682957090969219e-07,
+      "logits/chosen": -3.7294921875,
+      "logits/rejected": -3.7994141578674316,
+      "logps/chosen": -321.1875,
+      "logps/rejected": -299.26251220703125,
+      "loss": 0.6594,
+      "rewards/accuracies": 0.610156238079071,
+      "rewards/chosen": 0.826586902141571,
+      "rewards/margins": 0.7918456792831421,
+      "rewards/rejected": 0.03476562350988388,
+      "step": 450
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 70.97492127696606,
+      "learning_rate": 8.599517368789979e-07,
+      "logits/chosen": -3.7105469703674316,
+      "logits/rejected": -3.724609375,
+      "logps/chosen": -329.73748779296875,
+      "logps/rejected": -301.92498779296875,
+      "loss": 0.6381,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.893310546875,
+      "rewards/margins": 0.787426769733429,
+      "rewards/rejected": 0.10600586235523224,
+      "step": 460
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 113.4642445941095,
+      "learning_rate": 8.513942915725159e-07,
+      "logits/chosen": -3.7027344703674316,
+      "logits/rejected": -3.73046875,
+      "logps/chosen": -323.2749938964844,
+      "logps/rejected": -307.0,
+      "loss": 0.6326,
+      "rewards/accuracies": 0.632031261920929,
+      "rewards/chosen": 1.0454590320587158,
+      "rewards/margins": 0.903613269329071,
+      "rewards/rejected": 0.14177246391773224,
+      "step": 470
+    },
+    {
+      "epoch": 1.0041873855011778,
+      "grad_norm": 20.06602896361506,
+      "learning_rate": 8.426284482586396e-07,
+      "logits/chosen": -3.65625,
+      "logits/rejected": -3.8149349689483643,
+      "logps/chosen": -332.4934997558594,
+      "logps/rejected": -307.1428527832031,
+      "loss": 0.557,
+      "rewards/accuracies": 0.6783008575439453,
+      "rewards/chosen": 1.2439631223678589,
+      "rewards/margins": 1.2396240234375,
+      "rewards/rejected": 0.0042130108922719955,
+      "step": 480
+    },
+    {
+      "epoch": 1.0251243130070662,
+      "grad_norm": 29.33573218810576,
+      "learning_rate": 8.336594056111197e-07,
+      "logits/chosen": -3.791796922683716,
+      "logits/rejected": -3.8296875953674316,
+      "logps/chosen": -326.9624938964844,
+      "logps/rejected": -296.7250061035156,
+      "loss": 0.2376,
+      "rewards/accuracies": 0.899218738079071,
+      "rewards/chosen": 1.890234351158142,
+      "rewards/margins": 2.6279296875,
+      "rewards/rejected": -0.7376464605331421,
+      "step": 490
+    },
+    {
+      "epoch": 1.0460612405129548,
+      "grad_norm": 32.27685063622708,
+      "learning_rate": 8.244924828131667e-07,
+      "logits/chosen": -3.687695264816284,
+      "logits/rejected": -3.7779297828674316,
+      "logps/chosen": -333.01251220703125,
+      "logps/rejected": -312.25,
+      "loss": 0.2108,
+      "rewards/accuracies": 0.917187511920929,
+      "rewards/chosen": 2.0382323265075684,
+      "rewards/margins": 2.906054735183716,
+      "rewards/rejected": -0.8681396245956421,
+      "step": 500
+    },
+    {
+      "epoch": 1.0460612405129548,
+      "eval_logits/chosen": -3.70957350730896,
+      "eval_logits/rejected": -3.7688491344451904,
+      "eval_logps/chosen": -321.26983642578125,
+      "eval_logps/rejected": -301.1111145019531,
+      "eval_loss": 0.6046420931816101,
+      "eval_rewards/accuracies": 0.6388888955116272,
+      "eval_rewards/chosen": 1.037264347076416,
+      "eval_rewards/margins": 0.9335627555847168,
+      "eval_rewards/rejected": 0.1040116548538208,
+      "eval_runtime": 37.4726,
+      "eval_samples_per_second": 53.372,
+      "eval_steps_per_second": 1.681,
+      "step": 500
+    },
+    {
+      "epoch": 1.0669981680188432,
+      "grad_norm": 26.023306929092584,
+      "learning_rate": 8.151331164028543e-07,
+      "logits/chosen": -3.673046827316284,
+      "logits/rejected": -3.7652344703674316,
+      "logps/chosen": -330.17498779296875,
+      "logps/rejected": -300.125,
+      "loss": 0.2172,
+      "rewards/accuracies": 0.9117187261581421,
+      "rewards/chosen": 2.103076219558716,
+      "rewards/margins": 2.892871141433716,
+      "rewards/rejected": -0.79052734375,
+      "step": 510
+    },
+    {
+      "epoch": 1.0879350955247318,
+      "grad_norm": 27.67690055706761,
+      "learning_rate": 8.055868570489246e-07,
+      "logits/chosen": -3.6617188453674316,
+      "logits/rejected": -3.7353515625,
+      "logps/chosen": -330.73748779296875,
+      "logps/rejected": -308.01251220703125,
+      "loss": 0.2281,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": 2.0562987327575684,
+      "rewards/margins": 2.852832078933716,
+      "rewards/rejected": -0.7958008050918579,
+      "step": 520
+    },
+    {
+      "epoch": 1.1088720230306202,
+      "grad_norm": 47.18233049420054,
+      "learning_rate": 7.958593662589068e-07,
+      "logits/chosen": -3.6947264671325684,
+      "logits/rejected": -3.768749952316284,
+      "logps/chosen": -312.4750061035156,
+      "logps/rejected": -294.48748779296875,
+      "loss": 0.2215,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": 2.0047364234924316,
+      "rewards/margins": 2.7953124046325684,
+      "rewards/rejected": -0.790209949016571,
+      "step": 530
+    },
+    {
+      "epoch": 1.1298089505365088,
+      "grad_norm": 20.345020357654054,
+      "learning_rate": 7.859564130215016e-07,
+      "logits/chosen": -3.6585936546325684,
+      "logits/rejected": -3.740429639816284,
+      "logps/chosen": -328.4750061035156,
+      "logps/rejected": -285.98748779296875,
+      "loss": 0.2178,
+      "rewards/accuracies": 0.91015625,
+      "rewards/chosen": 2.0340332984924316,
+      "rewards/margins": 2.9429688453674316,
+      "rewards/rejected": -0.908007800579071,
+      "step": 540
+    },
+    {
+      "epoch": 1.1507458780423974,
+      "grad_norm": 20.38502035379127,
+      "learning_rate": 7.75883870385223e-07,
+      "logits/chosen": -3.715625047683716,
+      "logits/rejected": -3.800976514816284,
+      "logps/chosen": -327.95001220703125,
+      "logps/rejected": -303.6875,
+      "loss": 0.2166,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.032763719558716,
+      "rewards/margins": 3.011425733566284,
+      "rewards/rejected": -0.9779297113418579,
+      "step": 550
+    },
+    {
+      "epoch": 1.1716828055482857,
+      "grad_norm": 18.590334057828347,
+      "learning_rate": 7.656477119753267e-07,
+      "logits/chosen": -3.676953077316284,
+      "logits/rejected": -3.8255858421325684,
+      "logps/chosen": -340.0625,
+      "logps/rejected": -286.5,
+      "loss": 0.2086,
+      "rewards/accuracies": 0.913281261920929,
+      "rewards/chosen": 2.1092772483825684,
+      "rewards/margins": 2.95556640625,
+      "rewards/rejected": -0.845703125,
+      "step": 560
+    },
+    {
+      "epoch": 1.1926197330541743,
+      "grad_norm": 30.084682128330225,
+      "learning_rate": 7.552540084510894e-07,
+      "logits/chosen": -3.6587891578674316,
+      "logits/rejected": -3.7880859375,
+      "logps/chosen": -329.1875,
+      "logps/rejected": -299.5625,
+      "loss": 0.1983,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": 2.0482420921325684,
+      "rewards/margins": 3.0365233421325684,
+      "rewards/rejected": -0.987744152545929,
+      "step": 570
+    },
+    {
+      "epoch": 1.2135566605600627,
+      "grad_norm": 27.274056212881028,
+      "learning_rate": 7.447089239055427e-07,
+      "logits/chosen": -3.5718750953674316,
+      "logits/rejected": -3.723437547683716,
+      "logps/chosen": -328.5375061035156,
+      "logps/rejected": -303.01251220703125,
+      "loss": 0.2138,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": 2.203320264816284,
+      "rewards/margins": 3.1495118141174316,
+      "rewards/rejected": -0.947460949420929,
+      "step": 580
+    },
+    {
+      "epoch": 1.2344935880659513,
+      "grad_norm": 20.023367416268798,
+      "learning_rate": 7.340187122097929e-07,
+      "logits/chosen": -3.6322264671325684,
+      "logits/rejected": -3.7603516578674316,
+      "logps/chosen": -336.7250061035156,
+      "logps/rejected": -300.86248779296875,
+      "loss": 0.207,
+      "rewards/accuracies": 0.9140625,
+      "rewards/chosen": 2.274121046066284,
+      "rewards/margins": 3.1513671875,
+      "rewards/rejected": -0.8777831792831421,
+      "step": 590
+    },
+    {
+      "epoch": 1.25543051557184,
+      "grad_norm": 19.881350737567885,
+      "learning_rate": 7.231897133040996e-07,
+      "logits/chosen": -3.6463866233825684,
+      "logits/rejected": -3.7314453125,
+      "logps/chosen": -316.5249938964844,
+      "logps/rejected": -297.3374938964844,
+      "loss": 0.2047,
+      "rewards/accuracies": 0.917187511920929,
+      "rewards/chosen": 2.125195264816284,
+      "rewards/margins": 3.0887694358825684,
+      "rewards/rejected": -0.964770495891571,
+      "step": 600
+    },
+    {
+      "epoch": 1.25543051557184,
+      "eval_logits/chosen": -3.69394850730896,
+      "eval_logits/rejected": -3.752232074737549,
+      "eval_logps/chosen": -321.1745910644531,
+      "eval_logps/rejected": -301.1428527832031,
+      "eval_loss": 0.6215904355049133,
+      "eval_rewards/accuracies": 0.6354166865348816,
+      "eval_rewards/chosen": 1.0917038917541504,
+      "eval_rewards/margins": 1.0199652910232544,
+      "eval_rewards/rejected": 0.0719401016831398,
+      "eval_runtime": 37.4418,
+      "eval_samples_per_second": 53.416,
+      "eval_steps_per_second": 1.683,
+      "step": 600
+    },
+    {
+      "epoch": 1.2763674430777283,
+      "grad_norm": 29.808637867916268,
+      "learning_rate": 7.122283494379075e-07,
+      "logits/chosen": -3.639355421066284,
+      "logits/rejected": -3.718945264816284,
+      "logps/chosen": -328.57501220703125,
+      "logps/rejected": -294.6625061035156,
+      "loss": 0.2082,
+      "rewards/accuracies": 0.9085937738418579,
+      "rewards/chosen": 2.2525391578674316,
+      "rewards/margins": 3.189746141433716,
+      "rewards/rejected": -0.9357665777206421,
+      "step": 610
+    },
+    {
+      "epoch": 1.297304370583617,
+      "grad_norm": 26.1235410500105,
+      "learning_rate": 7.011411213610661e-07,
+      "logits/chosen": -3.716015577316284,
+      "logits/rejected": -3.770703077316284,
+      "logps/chosen": -327.8999938964844,
+      "logps/rejected": -293.4375,
+      "loss": 0.2135,
+      "rewards/accuracies": 0.9007812738418579,
+      "rewards/chosen": 2.216845750808716,
+      "rewards/margins": 3.1875,
+      "rewards/rejected": -0.970288097858429,
+      "step": 620
+    },
+    {
+      "epoch": 1.3182412980895053,
+      "grad_norm": 16.563477466236296,
+      "learning_rate": 6.899346044684928e-07,
+      "logits/chosen": -3.579882860183716,
+      "logits/rejected": -3.7505860328674316,
+      "logps/chosen": -338.8125,
+      "logps/rejected": -299.04998779296875,
+      "loss": 0.2088,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.2646484375,
+      "rewards/margins": 3.2215819358825684,
+      "rewards/rejected": -0.9583740234375,
+      "step": 630
+    },
+    {
+      "epoch": 1.339178225595394,
+      "grad_norm": 28.708949410921242,
+      "learning_rate": 6.786154449005663e-07,
+      "logits/chosen": -3.678515672683716,
+      "logits/rejected": -3.7865233421325684,
+      "logps/chosen": -329.42498779296875,
+      "logps/rejected": -283.0625,
+      "loss": 0.204,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.10986328125,
+      "rewards/margins": 3.197070360183716,
+      "rewards/rejected": -1.0874512195587158,
+      "step": 640
+    },
+    {
+      "epoch": 1.3601151531012823,
+      "grad_norm": 21.83884647546876,
+      "learning_rate": 6.671903556015663e-07,
+      "logits/chosen": -3.656054735183716,
+      "logits/rejected": -3.697460889816284,
+      "logps/chosen": -318.42498779296875,
+      "logps/rejected": -294.9750061035156,
+      "loss": 0.2112,
+      "rewards/accuracies": 0.9140625,
+      "rewards/chosen": 1.9429199695587158,
+      "rewards/margins": 3.1246094703674316,
+      "rewards/rejected": -1.1823241710662842,
+      "step": 650
+    },
+    {
+      "epoch": 1.3810520806071709,
+      "grad_norm": 19.65309577614754,
+      "learning_rate": 6.556661123384908e-07,
+      "logits/chosen": -3.6773438453674316,
+      "logits/rejected": -3.7906250953674316,
+      "logps/chosen": -335.8125,
+      "logps/rejected": -296.2250061035156,
+      "loss": 0.2065,
+      "rewards/accuracies": 0.89453125,
+      "rewards/chosen": 2.045849561691284,
+      "rewards/margins": 3.179980516433716,
+      "rewards/rejected": -1.1333496570587158,
+      "step": 660
+    },
+    {
+      "epoch": 1.4019890081130595,
+      "grad_norm": 27.548046428252114,
+      "learning_rate": 6.440495496826188e-07,
+      "logits/chosen": -3.7044920921325684,
+      "logits/rejected": -3.830273389816284,
+      "logps/chosen": -320.3374938964844,
+      "logps/rejected": -292.17498779296875,
+      "loss": 0.2076,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": 1.99951171875,
+      "rewards/margins": 3.008105516433716,
+      "rewards/rejected": -1.009252905845642,
+      "step": 670
+    },
+    {
+      "epoch": 1.4229259356189479,
+      "grad_norm": 20.734893383294562,
+      "learning_rate": 6.323475569561966e-07,
+      "logits/chosen": -3.7505860328674316,
+      "logits/rejected": -3.7914061546325684,
+      "logps/chosen": -336.95623779296875,
+      "logps/rejected": -309.20001220703125,
+      "loss": 0.2147,
+      "rewards/accuracies": 0.901562511920929,
+      "rewards/chosen": 2.1971192359924316,
+      "rewards/margins": 3.1949219703674316,
+      "rewards/rejected": -0.997851550579071,
+      "step": 680
+    },
+    {
+      "epoch": 1.4438628631248365,
+      "grad_norm": 21.72197810144563,
+      "learning_rate": 6.205670741466554e-07,
+      "logits/chosen": -3.6890625953674316,
+      "logits/rejected": -3.8197264671325684,
+      "logps/chosen": -322.625,
+      "logps/rejected": -293.32501220703125,
+      "loss": 0.208,
+      "rewards/accuracies": 0.9140625,
+      "rewards/chosen": 2.0825438499450684,
+      "rewards/margins": 3.232714891433716,
+      "rewards/rejected": -1.1504638195037842,
+      "step": 690
+    },
+    {
+      "epoch": 1.464799790630725,
+      "grad_norm": 17.32564942850288,
+      "learning_rate": 6.087150877907785e-07,
+      "logits/chosen": -3.702343702316284,
+      "logits/rejected": -3.779296875,
+      "logps/chosen": -328.07501220703125,
+      "logps/rejected": -306.5,
+      "loss": 0.2093,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 2.227001905441284,
+      "rewards/margins": 3.344043016433716,
+      "rewards/rejected": -1.1149170398712158,
+      "step": 700
+    },
+    {
+      "epoch": 1.464799790630725,
+      "eval_logits/chosen": -3.6971726417541504,
+      "eval_logits/rejected": -3.757936477661133,
+      "eval_logps/chosen": -321.25396728515625,
+      "eval_logps/rejected": -301.30157470703125,
+      "eval_loss": 0.6190463900566101,
+      "eval_rewards/accuracies": 0.6458333134651184,
+      "eval_rewards/chosen": 1.0818452835083008,
+      "eval_rewards/margins": 1.0839533805847168,
+      "eval_rewards/rejected": -0.0016896080924198031,
+      "eval_runtime": 37.4822,
+      "eval_samples_per_second": 53.359,
+      "eval_steps_per_second": 1.681,
+      "step": 700
+    },
+    {
+      "epoch": 1.4857367181366135,
+      "grad_norm": 34.495322737904495,
+      "learning_rate": 5.96798626831265e-07,
+      "logits/chosen": -3.6705079078674316,
+      "logits/rejected": -3.8187499046325684,
+      "logps/chosen": -328.73748779296875,
+      "logps/rejected": -298.6000061035156,
+      "loss": 0.2044,
+      "rewards/accuracies": 0.914843738079071,
+      "rewards/chosen": 2.0855956077575684,
+      "rewards/margins": 3.2300782203674316,
+      "rewards/rejected": -1.14306640625,
+      "step": 710
+    },
+    {
+      "epoch": 1.5066736456425018,
+      "grad_norm": 21.039592880809614,
+      "learning_rate": 5.848247584481423e-07,
+      "logits/chosen": -3.7105469703674316,
+      "logits/rejected": -3.755859375,
+      "logps/chosen": -335.78125,
+      "logps/rejected": -298.0874938964844,
+      "loss": 0.212,
+      "rewards/accuracies": 0.917187511920929,
+      "rewards/chosen": 2.1192870140075684,
+      "rewards/margins": 3.139941453933716,
+      "rewards/rejected": -1.020263671875,
+      "step": 720
+    },
+    {
+      "epoch": 1.5276105731483904,
+      "grad_norm": 32.47201089417023,
+      "learning_rate": 5.728005838675025e-07,
+      "logits/chosen": -3.630664110183716,
+      "logits/rejected": -3.740234375,
+      "logps/chosen": -311.48748779296875,
+      "logps/rejected": -291.92498779296875,
+      "loss": 0.2306,
+      "rewards/accuracies": 0.889843761920929,
+      "rewards/chosen": 2.0782227516174316,
+      "rewards/margins": 3.0840821266174316,
+      "rewards/rejected": -1.004614233970642,
+      "step": 730
+    },
+    {
+      "epoch": 1.548547500654279,
+      "grad_norm": 23.237237873523405,
+      "learning_rate": 5.607332341500462e-07,
+      "logits/chosen": -3.725781202316284,
+      "logits/rejected": -3.788867235183716,
+      "logps/chosen": -341.2749938964844,
+      "logps/rejected": -303.2250061035156,
+      "loss": 0.2196,
+      "rewards/accuracies": 0.9007812738418579,
+      "rewards/chosen": 2.274121046066284,
+      "rewards/margins": 3.3619141578674316,
+      "rewards/rejected": -1.0873534679412842,
+      "step": 740
+    },
+    {
+      "epoch": 1.5694844281601674,
+      "grad_norm": 16.96184668772172,
+      "learning_rate": 5.486298659619345e-07,
+      "logits/chosen": -3.6343750953674316,
+      "logits/rejected": -3.7005858421325684,
+      "logps/chosen": -339.6875,
+      "logps/rejected": -308.51251220703125,
+      "loss": 0.211,
+      "rewards/accuracies": 0.9046875238418579,
+      "rewards/chosen": 2.29931640625,
+      "rewards/margins": 3.382128953933716,
+      "rewards/rejected": -1.0826904773712158,
+      "step": 750
+    },
+    {
+      "epoch": 1.590421355666056,
+      "grad_norm": 22.358070461719336,
+      "learning_rate": 5.364976573304537e-07,
+      "logits/chosen": -3.618945360183716,
+      "logits/rejected": -3.7896485328674316,
+      "logps/chosen": -330.48748779296875,
+      "logps/rejected": -294.3125,
+      "loss": 0.2046,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": 2.264843702316284,
+      "rewards/margins": 3.3062500953674316,
+      "rewards/rejected": -1.04248046875,
+      "step": 760
+    },
+    {
+      "epoch": 1.6113582831719446,
+      "grad_norm": 25.16187627344586,
+      "learning_rate": 5.243438033870126e-07,
+      "logits/chosen": -3.6978516578674316,
+      "logits/rejected": -3.7783203125,
+      "logps/chosen": -322.9125061035156,
+      "logps/rejected": -297.92498779296875,
+      "loss": 0.2104,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": 2.1826171875,
+      "rewards/margins": 3.22900390625,
+      "rewards/rejected": -1.0475342273712158,
+      "step": 770
+    },
+    {
+      "epoch": 1.632295210677833,
+      "grad_norm": 22.13377813501472,
+      "learning_rate": 5.121755120999948e-07,
+      "logits/chosen": -3.712890625,
+      "logits/rejected": -3.7740235328674316,
+      "logps/chosen": -314.75,
+      "logps/rejected": -297.9375,
+      "loss": 0.2175,
+      "rewards/accuracies": 0.9078124761581421,
+      "rewards/chosen": 2.0287108421325684,
+      "rewards/margins": 3.209716796875,
+      "rewards/rejected": -1.181396484375,
+      "step": 780
+    },
+    {
+      "epoch": 1.6532321381837214,
+      "grad_norm": 25.854603996816074,
+      "learning_rate": 5e-07,
+      "logits/chosen": -3.6947264671325684,
+      "logits/rejected": -3.783203125,
+      "logps/chosen": -320.1000061035156,
+      "logps/rejected": -281.2875061035156,
+      "loss": 0.2265,
+      "rewards/accuracies": 0.8984375,
+      "rewards/chosen": 2.0169920921325684,
+      "rewards/margins": 3.0157227516174316,
+      "rewards/rejected": -0.9986572265625,
+      "step": 790
+    },
+    {
+      "epoch": 1.6741690656896102,
+      "grad_norm": 17.994500106345612,
+      "learning_rate": 4.878244879000052e-07,
+      "logits/chosen": -3.697460889816284,
+      "logits/rejected": -3.728320360183716,
+      "logps/chosen": -317.5625,
+      "logps/rejected": -300.1937561035156,
+      "loss": 0.2155,
+      "rewards/accuracies": 0.914843738079071,
+      "rewards/chosen": 2.0946288108825684,
+      "rewards/margins": 3.312695264816284,
+      "rewards/rejected": -1.217871069908142,
+      "step": 800
+    },
+    {
+      "epoch": 1.6741690656896102,
+      "eval_logits/chosen": -3.6877479553222656,
+      "eval_logits/rejected": -3.746279716491699,
+      "eval_logps/chosen": -321.25396728515625,
+      "eval_logps/rejected": -301.4444580078125,
+      "eval_loss": 0.6213928461074829,
+      "eval_rewards/accuracies": 0.6498016119003296,
+      "eval_rewards/chosen": 1.0270336866378784,
+      "eval_rewards/margins": 1.0708705186843872,
+      "eval_rewards/rejected": -0.0434182770550251,
+      "eval_runtime": 37.5335,
+      "eval_samples_per_second": 53.286,
+      "eval_steps_per_second": 1.679,
+      "step": 800
+    },
+    {
+      "epoch": 1.6951059931954986,
+      "grad_norm": 21.143141506330878,
+      "learning_rate": 4.756561966129875e-07,
+      "logits/chosen": -3.703320264816284,
+      "logits/rejected": -3.7837891578674316,
+      "logps/chosen": -342.5375061035156,
+      "logps/rejected": -300.45001220703125,
+      "loss": 0.2131,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": 2.2383790016174316,
+      "rewards/margins": 3.3817381858825684,
+      "rewards/rejected": -1.142602562904358,
+      "step": 810
+    },
+    {
+      "epoch": 1.716042920701387,
+      "grad_norm": 27.839814204813557,
+      "learning_rate": 4.635023426695462e-07,
+      "logits/chosen": -3.6732420921325684,
+      "logits/rejected": -3.779101610183716,
+      "logps/chosen": -329.0,
+      "logps/rejected": -292.0375061035156,
+      "loss": 0.1981,
+      "rewards/accuracies": 0.922656238079071,
+      "rewards/chosen": 2.10546875,
+      "rewards/margins": 3.2935547828674316,
+      "rewards/rejected": -1.1874511241912842,
+      "step": 820
+    },
+    {
+      "epoch": 1.7369798482072756,
+      "grad_norm": 27.11384250877626,
+      "learning_rate": 4.513701340380654e-07,
+      "logits/chosen": -3.7578125,
+      "logits/rejected": -3.8638672828674316,
+      "logps/chosen": -335.1625061035156,
+      "logps/rejected": -304.2875061035156,
+      "loss": 0.1966,
+      "rewards/accuracies": 0.921093761920929,
+      "rewards/chosen": 2.2708983421325684,
+      "rewards/margins": 3.421191453933716,
+      "rewards/rejected": -1.1507568359375,
+      "step": 830
+    },
+    {
+      "epoch": 1.7579167757131642,
+      "grad_norm": 19.63722343047291,
+      "learning_rate": 4.392667658499539e-07,
+      "logits/chosen": -3.6552734375,
+      "logits/rejected": -3.7802734375,
+      "logps/chosen": -325.38751220703125,
+      "logps/rejected": -295.3500061035156,
+      "loss": 0.2101,
+      "rewards/accuracies": 0.9203125238418579,
+      "rewards/chosen": 2.174121141433716,
+      "rewards/margins": 3.3104491233825684,
+      "rewards/rejected": -1.1359374523162842,
+      "step": 840
+    },
+    {
+      "epoch": 1.7788537032190526,
+      "grad_norm": 32.01859346300431,
+      "learning_rate": 4.2719941613249767e-07,
+      "logits/chosen": -3.694531202316284,
+      "logits/rejected": -3.802929639816284,
+      "logps/chosen": -340.42498779296875,
+      "logps/rejected": -303.20001220703125,
+      "loss": 0.1981,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.2837891578674316,
+      "rewards/margins": 3.420605421066284,
+      "rewards/rejected": -1.136816382408142,
+      "step": 850
+    },
+    {
+      "epoch": 1.7997906307249412,
+      "grad_norm": 19.834989579300487,
+      "learning_rate": 4.1517524155185767e-07,
+      "logits/chosen": -3.686718702316284,
+      "logits/rejected": -3.747265577316284,
+      "logps/chosen": -307.32501220703125,
+      "logps/rejected": -292.1499938964844,
+      "loss": 0.2067,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 2.0224609375,
+      "rewards/margins": 3.186718702316284,
+      "rewards/rejected": -1.164331078529358,
+      "step": 860
+    },
+    {
+      "epoch": 1.8207275582308298,
+      "grad_norm": 16.81808072026082,
+      "learning_rate": 4.03201373168735e-07,
+      "logits/chosen": -3.6781249046325684,
+      "logits/rejected": -3.791210889816284,
+      "logps/chosen": -335.61248779296875,
+      "logps/rejected": -298.5249938964844,
+      "loss": 0.1892,
+      "rewards/accuracies": 0.91796875,
+      "rewards/chosen": 2.25927734375,
+      "rewards/margins": 3.9580078125,
+      "rewards/rejected": -1.6982421875,
+      "step": 870
+    },
+    {
+      "epoch": 1.8416644857367181,
+      "grad_norm": 25.35803723271088,
+      "learning_rate": 3.9128491220922156e-07,
+      "logits/chosen": -3.601757764816284,
+      "logits/rejected": -3.706835985183716,
+      "logps/chosen": -323.86248779296875,
+      "logps/rejected": -297.64373779296875,
+      "loss": 0.2061,
+      "rewards/accuracies": 0.910937488079071,
+      "rewards/chosen": 2.234814405441284,
+      "rewards/margins": 3.3687500953674316,
+      "rewards/rejected": -1.133764624595642,
+      "step": 880
+    },
+    {
+      "epoch": 1.8626014132426065,
+      "grad_norm": 24.13879403591726,
+      "learning_rate": 3.794329258533446e-07,
+      "logits/chosen": -3.622265577316284,
+      "logits/rejected": -3.740039110183716,
+      "logps/chosen": -320.98748779296875,
+      "logps/rejected": -305.3374938964844,
+      "loss": 0.2035,
+      "rewards/accuracies": 0.914843738079071,
+      "rewards/chosen": 2.138867139816284,
+      "rewards/margins": 3.3033204078674316,
+      "rewards/rejected": -1.1640136241912842,
+      "step": 890
+    },
+    {
+      "epoch": 1.8835383407484951,
+      "grad_norm": 28.15584093000681,
+      "learning_rate": 3.6765244304380315e-07,
+      "logits/chosen": -3.701171875,
+      "logits/rejected": -3.8037109375,
+      "logps/chosen": -334.17498779296875,
+      "logps/rejected": -293.11248779296875,
+      "loss": 0.1903,
+      "rewards/accuracies": 0.9164062738418579,
+      "rewards/chosen": 2.214550733566284,
+      "rewards/margins": 3.4410157203674316,
+      "rewards/rejected": -1.226415991783142,
+      "step": 900
+    },
+    {
+      "epoch": 1.8835383407484951,
+      "eval_logits/chosen": -3.6879961490631104,
+      "eval_logits/rejected": -3.745039701461792,
+      "eval_logps/chosen": -321.3809509277344,
+      "eval_logps/rejected": -301.5396728515625,
+      "eval_loss": 0.6117067933082581,
+      "eval_rewards/accuracies": 0.6498016119003296,
+      "eval_rewards/chosen": 0.9993179440498352,
+      "eval_rewards/margins": 1.1053757667541504,
+      "eval_rewards/rejected": -0.1059182807803154,
+      "eval_runtime": 37.4132,
+      "eval_samples_per_second": 53.457,
+      "eval_steps_per_second": 1.684,
+      "step": 900
+    },
+    {
+      "epoch": 1.9044752682543837,
+      "grad_norm": 22.144304676783836,
+      "learning_rate": 3.559504503173812e-07,
+      "logits/chosen": -3.7134766578674316,
+      "logits/rejected": -3.791796922683716,
+      "logps/chosen": -329.2124938964844,
+      "logps/rejected": -301.2749938964844,
+      "loss": 0.1802,
+      "rewards/accuracies": 0.9203125238418579,
+      "rewards/chosen": 2.227246046066284,
+      "rewards/margins": 3.459179639816284,
+      "rewards/rejected": -1.2341797351837158,
+      "step": 910
+    },
+    {
+      "epoch": 1.9254121957602721,
+      "grad_norm": 22.922306037242627,
+      "learning_rate": 3.4433388766150915e-07,
+      "logits/chosen": -3.6216797828674316,
+      "logits/rejected": -3.767773389816284,
+      "logps/chosen": -322.875,
+      "logps/rejected": -294.13751220703125,
+      "loss": 0.2039,
+      "rewards/accuracies": 0.9039062261581421,
+      "rewards/chosen": 2.1859374046325684,
+      "rewards/margins": 3.30810546875,
+      "rewards/rejected": -1.121923804283142,
+      "step": 920
+    },
+    {
+      "epoch": 1.9463491232661607,
+      "grad_norm": 25.816710779987066,
+      "learning_rate": 3.328096443984337e-07,
+      "logits/chosen": -3.5894532203674316,
+      "logits/rejected": -3.650390625,
+      "logps/chosen": -330.61248779296875,
+      "logps/rejected": -299.875,
+      "loss": 0.203,
+      "rewards/accuracies": 0.910937488079071,
+      "rewards/chosen": 2.191943407058716,
+      "rewards/margins": 3.5072264671325684,
+      "rewards/rejected": -1.31396484375,
+      "step": 930
+    },
+    {
+      "epoch": 1.9672860507720493,
+      "grad_norm": 22.801911429334652,
+      "learning_rate": 3.213845550994336e-07,
+      "logits/chosen": -3.701367139816284,
+      "logits/rejected": -3.7607421875,
+      "logps/chosen": -328.2749938964844,
+      "logps/rejected": -318.17498779296875,
+      "loss": 0.2081,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.1390137672424316,
+      "rewards/margins": 3.433300733566284,
+      "rewards/rejected": -1.2932860851287842,
+      "step": 940
+    },
+    {
+      "epoch": 1.9882229782779377,
+      "grad_norm": 22.3373790994134,
+      "learning_rate": 3.1006539553150724e-07,
+      "logits/chosen": -3.7308592796325684,
+      "logits/rejected": -3.792187452316284,
+      "logps/chosen": -338.54998779296875,
+      "logps/rejected": -297.36248779296875,
+      "loss": 0.2003,
+      "rewards/accuracies": 0.907031238079071,
+      "rewards/chosen": 2.15087890625,
+      "rewards/margins": 3.355761766433716,
+      "rewards/rejected": -1.2052490711212158,
+      "step": 950
+    },
+    {
+      "epoch": 2.0083747710023556,
+      "grad_norm": 8.055383028270391,
+      "learning_rate": 2.9885887863893386e-07,
+      "logits/chosen": -3.6353490352630615,
+      "logits/rejected": -3.7562906742095947,
+      "logps/chosen": -334.98699951171875,
+      "logps/rejected": -295.8961181640625,
+      "loss": 0.1626,
+      "rewards/accuracies": 0.9349567294120789,
+      "rewards/chosen": 2.322544574737549,
+      "rewards/margins": 3.7064731121063232,
+      "rewards/rejected": -1.3828632831573486,
+      "step": 960
+    },
+    {
+      "epoch": 2.029311698508244,
+      "grad_norm": 7.372010575225488,
+      "learning_rate": 2.8777165056209254e-07,
+      "logits/chosen": -3.7425780296325684,
+      "logits/rejected": -3.8115234375,
+      "logps/chosen": -320.98126220703125,
+      "logps/rejected": -291.84375,
+      "loss": 0.0857,
+      "rewards/accuracies": 0.98046875,
+      "rewards/chosen": 2.6356444358825684,
+      "rewards/margins": 4.386132717132568,
+      "rewards/rejected": -1.7499511241912842,
+      "step": 970
+    },
+    {
+      "epoch": 2.0502486260141324,
+      "grad_norm": 6.810009686434846,
+      "learning_rate": 2.7681028669590033e-07,
+      "logits/chosen": -3.5927734375,
+      "logits/rejected": -3.714648485183716,
+      "logps/chosen": -340.9375,
+      "logps/rejected": -294.6000061035156,
+      "loss": 0.0827,
+      "rewards/accuracies": 0.9828125238418579,
+      "rewards/chosen": 2.8119139671325684,
+      "rewards/margins": 4.465624809265137,
+      "rewards/rejected": -1.654394507408142,
+      "step": 980
+    },
+    {
+      "epoch": 2.0711855535200208,
+      "grad_norm": 8.274714792991256,
+      "learning_rate": 2.659812877902069e-07,
+      "logits/chosen": -3.6654295921325684,
+      "logits/rejected": -3.7710938453674316,
+      "logps/chosen": -323.2875061035156,
+      "logps/rejected": -302.2250061035156,
+      "loss": 0.0928,
+      "rewards/accuracies": 0.975781261920929,
+      "rewards/chosen": 2.6698241233825684,
+      "rewards/margins": 4.3955078125,
+      "rewards/rejected": -1.7255859375,
+      "step": 990
+    },
+    {
+      "epoch": 2.0921224810259096,
+      "grad_norm": 8.883576628284072,
+      "learning_rate": 2.5529107609445735e-07,
+      "logits/chosen": -3.6298828125,
+      "logits/rejected": -3.7630858421325684,
+      "logps/chosen": -336.2124938964844,
+      "logps/rejected": -307.76251220703125,
+      "loss": 0.0891,
+      "rewards/accuracies": 0.9710937738418579,
+      "rewards/chosen": 2.733691453933716,
+      "rewards/margins": 4.4208984375,
+      "rewards/rejected": -1.6874511241912842,
+      "step": 1000
+    },
+    {
+      "epoch": 2.0921224810259096,
+      "eval_logits/chosen": -3.680555582046509,
+      "eval_logits/rejected": -3.738591194152832,
+      "eval_logps/chosen": -321.20635986328125,
+      "eval_logps/rejected": -301.6031799316406,
+      "eval_loss": 0.6198596358299255,
+      "eval_rewards/accuracies": 0.658730149269104,
+      "eval_rewards/chosen": 1.0863715410232544,
+      "eval_rewards/margins": 1.2334449291229248,
+      "eval_rewards/rejected": -0.1471199095249176,
+      "eval_runtime": 37.5194,
+      "eval_samples_per_second": 53.306,
+      "eval_steps_per_second": 1.679,
+      "step": 1000
+    },
+    {
+      "epoch": 2.113059408531798,
+      "grad_norm": 10.065711919045375,
+      "learning_rate": 2.447459915489106e-07,
+      "logits/chosen": -3.6693358421325684,
+      "logits/rejected": -3.7816405296325684,
+      "logps/chosen": -319.1875,
+      "logps/rejected": -293.3999938964844,
+      "loss": 0.0905,
+      "rewards/accuracies": 0.975781261920929,
+      "rewards/chosen": 2.769726514816284,
+      "rewards/margins": 4.363867282867432,
+      "rewards/rejected": -1.594335913658142,
+      "step": 1010
+    },
+    {
+      "epoch": 2.1339963360376863,
+      "grad_norm": 8.457705733501081,
+      "learning_rate": 2.3435228802467337e-07,
+      "logits/chosen": -3.7275390625,
+      "logits/rejected": -3.786914110183716,
+      "logps/chosen": -324.5249938964844,
+      "logps/rejected": -286.67498779296875,
+      "loss": 0.0862,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": 2.6739258766174316,
+      "rewards/margins": 4.3798828125,
+      "rewards/rejected": -1.706201195716858,
+      "step": 1020
+    },
+    {
+      "epoch": 2.154933263543575,
+      "grad_norm": 9.996043624676643,
+      "learning_rate": 2.24116129614777e-07,
+      "logits/chosen": -3.654296875,
+      "logits/rejected": -3.7123045921325684,
+      "logps/chosen": -320.01251220703125,
+      "logps/rejected": -298.7250061035156,
+      "loss": 0.084,
+      "rewards/accuracies": 0.9789062738418579,
+      "rewards/chosen": 2.6923828125,
+      "rewards/margins": 4.448828220367432,
+      "rewards/rejected": -1.7559082508087158,
+      "step": 1030
+    },
+    {
+      "epoch": 2.1758701910494636,
+      "grad_norm": 7.990049871558863,
+      "learning_rate": 2.1404358697849856e-07,
+      "logits/chosen": -3.658007860183716,
+      "logits/rejected": -3.730664014816284,
+      "logps/chosen": -339.7250061035156,
+      "logps/rejected": -297.73748779296875,
+      "loss": 0.0877,
+      "rewards/accuracies": 0.9765625,
+      "rewards/chosen": 2.793164014816284,
+      "rewards/margins": 4.4267578125,
+      "rewards/rejected": -1.6344726085662842,
+      "step": 1040
+    },
+    {
+      "epoch": 2.196807118555352,
+      "grad_norm": 7.82554815006052,
+      "learning_rate": 2.0414063374109326e-07,
+      "logits/chosen": -3.7339844703674316,
+      "logits/rejected": -3.817187547683716,
+      "logps/chosen": -335.42498779296875,
+      "logps/rejected": -296.82501220703125,
+      "loss": 0.0855,
+      "rewards/accuracies": 0.98046875,
+      "rewards/chosen": 2.8397459983825684,
+      "rewards/margins": 4.5947265625,
+      "rewards/rejected": -1.755126953125,
+      "step": 1050
+    },
+    {
+      "epoch": 2.2177440460612403,
+      "grad_norm": 9.376381281513892,
+      "learning_rate": 1.9441314295107535e-07,
+      "logits/chosen": -3.6865234375,
+      "logits/rejected": -3.7669920921325684,
+      "logps/chosen": -318.76251220703125,
+      "logps/rejected": -305.3125,
+      "loss": 0.0866,
+      "rewards/accuracies": 0.9789062738418579,
+      "rewards/chosen": 2.767285108566284,
+      "rewards/margins": 4.5673828125,
+      "rewards/rejected": -1.802392601966858,
+      "step": 1060
+    },
+    {
+      "epoch": 2.238680973567129,
+      "grad_norm": 8.876269496524813,
+      "learning_rate": 1.8486688359714564e-07,
+      "logits/chosen": -3.7171874046325684,
+      "logits/rejected": -3.8031249046325684,
+      "logps/chosen": -321.6625061035156,
+      "logps/rejected": -294.9125061035156,
+      "loss": 0.0908,
+      "rewards/accuracies": 0.977343738079071,
+      "rewards/chosen": 2.8726563453674316,
+      "rewards/margins": 4.5537109375,
+      "rewards/rejected": -1.680810570716858,
+      "step": 1070
+    },
+    {
+      "epoch": 2.2596179010730175,
+      "grad_norm": 7.426029841180805,
+      "learning_rate": 1.7550751718683337e-07,
+      "logits/chosen": -3.662890672683716,
+      "logits/rejected": -3.7818360328674316,
+      "logps/chosen": -322.9125061035156,
+      "logps/rejected": -309.38751220703125,
+      "loss": 0.0815,
+      "rewards/accuracies": 0.98046875,
+      "rewards/chosen": 2.8470702171325684,
+      "rewards/margins": 4.602734565734863,
+      "rewards/rejected": -1.7553222179412842,
+      "step": 1080
+    },
+    {
+      "epoch": 2.280554828578906,
+      "grad_norm": 6.709710268206985,
+      "learning_rate": 1.6634059438888032e-07,
+      "logits/chosen": -3.6224608421325684,
+      "logits/rejected": -3.8003907203674316,
+      "logps/chosen": -331.70001220703125,
+      "logps/rejected": -294.70001220703125,
+      "loss": 0.0792,
+      "rewards/accuracies": 0.9828125238418579,
+      "rewards/chosen": 2.777050733566284,
+      "rewards/margins": 4.439453125,
+      "rewards/rejected": -1.6633789539337158,
+      "step": 1090
+    },
+    {
+      "epoch": 2.3014917560847947,
+      "grad_norm": 7.075190343671591,
+      "learning_rate": 1.573715517413604e-07,
+      "logits/chosen": -3.7349610328674316,
+      "logits/rejected": -3.7867188453674316,
+      "logps/chosen": -326.73748779296875,
+      "logps/rejected": -289.92498779296875,
+      "loss": 0.0841,
+      "rewards/accuracies": 0.9820312261581421,
+      "rewards/chosen": 2.744433641433716,
+      "rewards/margins": 4.492383003234863,
+      "rewards/rejected": -1.747656226158142,
+      "step": 1100
+    },
+    {
+      "epoch": 2.3014917560847947,
+      "eval_logits/chosen": -3.679811477661133,
+      "eval_logits/rejected": -3.7403273582458496,
+      "eval_logps/chosen": -321.1269836425781,
+      "eval_logps/rejected": -301.6031799316406,
+      "eval_loss": 0.6190533638000488,
+      "eval_rewards/accuracies": 0.6622023582458496,
+      "eval_rewards/chosen": 1.1681857109069824,
+      "eval_rewards/margins": 1.3038194179534912,
+      "eval_rewards/rejected": -0.1355406790971756,
+      "eval_runtime": 37.4755,
+      "eval_samples_per_second": 53.368,
+      "eval_steps_per_second": 1.681,
+      "step": 1100
+    },
+    {
+      "epoch": 2.322428683590683,
+      "grad_norm": 7.6472971292571685,
+      "learning_rate": 1.486057084274841e-07,
+      "logits/chosen": -3.689453125,
+      "logits/rejected": -3.728515625,
+      "logps/chosen": -315.76251220703125,
+      "logps/rejected": -300.6499938964844,
+      "loss": 0.0811,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.755175828933716,
+      "rewards/margins": 4.532031059265137,
+      "rewards/rejected": -1.775537133216858,
+      "step": 1110
+    },
+    {
+      "epoch": 2.3433656110965715,
+      "grad_norm": 8.434313746119711,
+      "learning_rate": 1.4004826312100216e-07,
+      "logits/chosen": -3.646679639816284,
+      "logits/rejected": -3.728320360183716,
+      "logps/chosen": -339.5,
+      "logps/rejected": -307.8374938964844,
+      "loss": 0.0813,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": 2.924023389816284,
+      "rewards/margins": 4.760156154632568,
+      "rewards/rejected": -1.8360595703125,
+      "step": 1120
+    },
+    {
+      "epoch": 2.36430253860246,
+      "grad_norm": 7.182620905123345,
+      "learning_rate": 1.3170429090307822e-07,
+      "logits/chosen": -3.715039014816284,
+      "logits/rejected": -3.859570264816284,
+      "logps/chosen": -337.01251220703125,
+      "logps/rejected": -294.29998779296875,
+      "loss": 0.0782,
+      "rewards/accuracies": 0.98046875,
+      "rewards/chosen": 2.99609375,
+      "rewards/margins": 4.679491996765137,
+      "rewards/rejected": -1.6840331554412842,
+      "step": 1130
+    },
+    {
+      "epoch": 2.3852394661083487,
+      "grad_norm": 6.811215705617649,
+      "learning_rate": 1.235787402524603e-07,
+      "logits/chosen": -3.646289110183716,
+      "logits/rejected": -3.704882860183716,
+      "logps/chosen": -326.2250061035156,
+      "logps/rejected": -300.5874938964844,
+      "loss": 0.0837,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 2.941113233566284,
+      "rewards/margins": 4.600390434265137,
+      "rewards/rejected": -1.65771484375,
+      "step": 1140
+    },
+    {
+      "epoch": 2.406176393614237,
+      "grad_norm": 8.100397473265545,
+      "learning_rate": 1.1567643011073391e-07,
+      "logits/chosen": -3.6865234375,
+      "logits/rejected": -3.7154297828674316,
+      "logps/chosen": -315.88751220703125,
+      "logps/rejected": -304.11248779296875,
+      "loss": 0.0824,
+      "rewards/accuracies": 0.979687511920929,
+      "rewards/chosen": 2.908007860183716,
+      "rewards/margins": 4.601953029632568,
+      "rewards/rejected": -1.694433569908142,
+      "step": 1150
+    },
+    {
+      "epoch": 2.4271133211201255,
+      "grad_norm": 9.025672602055698,
+      "learning_rate": 1.0800204702439935e-07,
+      "logits/chosen": -3.682812452316284,
+      "logits/rejected": -3.8056640625,
+      "logps/chosen": -328.0249938964844,
+      "logps/rejected": -296.125,
+      "loss": 0.0831,
+      "rewards/accuracies": 0.9789062738418579,
+      "rewards/chosen": 2.8373045921325684,
+      "rewards/margins": 4.632031440734863,
+      "rewards/rejected": -1.7949707508087158,
+      "step": 1160
+    },
+    {
+      "epoch": 2.4480502486260143,
+      "grad_norm": 6.9055857094158934,
+      "learning_rate": 1.0056014236546645e-07,
+      "logits/chosen": -3.6509766578674316,
+      "logits/rejected": -3.728515625,
+      "logps/chosen": -334.73748779296875,
+      "logps/rejected": -302.6625061035156,
+      "loss": 0.0908,
+      "rewards/accuracies": 0.971875011920929,
+      "rewards/chosen": 2.9486327171325684,
+      "rewards/margins": 4.603906154632568,
+      "rewards/rejected": -1.655175805091858,
+      "step": 1170
+    },
+    {
+      "epoch": 2.4689871761319027,
+      "grad_norm": 7.003813374150082,
+      "learning_rate": 9.335512963221731e-08,
+      "logits/chosen": -3.666796922683716,
+      "logits/rejected": -3.7515625953674316,
+      "logps/chosen": -329.9125061035156,
+      "logps/rejected": -299.75,
+      "loss": 0.087,
+      "rewards/accuracies": 0.9710937738418579,
+      "rewards/chosen": 2.873730421066284,
+      "rewards/margins": 4.641797065734863,
+      "rewards/rejected": -1.7666015625,
+      "step": 1180
+    },
+    {
+      "epoch": 2.489924103637791,
+      "grad_norm": 7.85231498826632,
+      "learning_rate": 8.639128183173516e-08,
+      "logits/chosen": -3.716796875,
+      "logits/rejected": -3.8199219703674316,
+      "logps/chosen": -323.70001220703125,
+      "logps/rejected": -294.4375,
+      "loss": 0.086,
+      "rewards/accuracies": 0.9765625,
+      "rewards/chosen": 2.7681641578674316,
+      "rewards/margins": 4.541015625,
+      "rewards/rejected": -1.772705078125,
+      "step": 1190
+    },
+    {
+      "epoch": 2.51086103114368,
+      "grad_norm": 6.804506658470371,
+      "learning_rate": 7.967272894575311e-08,
+      "logits/chosen": -3.7041015625,
+      "logits/rejected": -3.7933592796325684,
+      "logps/chosen": -312.17498779296875,
+      "logps/rejected": -301.7875061035156,
+      "loss": 0.0889,
+      "rewards/accuracies": 0.9765625,
+      "rewards/chosen": 2.762500047683716,
+      "rewards/margins": 4.5361328125,
+      "rewards/rejected": -1.7748534679412842,
+      "step": 1200
+    },
+    {
+      "epoch": 2.51086103114368,
+      "eval_logits/chosen": -3.676835298538208,
+      "eval_logits/rejected": -3.735863208770752,
+      "eval_logps/chosen": -321.1269836425781,
+      "eval_logps/rejected": -301.6031799316406,
+      "eval_loss": 0.624972403049469,
+      "eval_rewards/accuracies": 0.6617063283920288,
+      "eval_rewards/chosen": 1.1719679832458496,
+      "eval_rewards/margins": 1.3187934160232544,
+      "eval_rewards/rejected": -0.14705790579319,
+      "eval_runtime": 37.4374,
+      "eval_samples_per_second": 53.423,
+      "eval_steps_per_second": 1.683,
+      "step": 1200
+    },
+    {
+      "epoch": 2.5317979586495682,
+      "grad_norm": 6.785952534103185,
+      "learning_rate": 7.320345548132678e-08,
+      "logits/chosen": -3.6380858421325684,
+      "logits/rejected": -3.7484374046325684,
+      "logps/chosen": -316.45001220703125,
+      "logps/rejected": -293.20001220703125,
+      "loss": 0.0846,
+      "rewards/accuracies": 0.975781261920929,
+      "rewards/chosen": 2.853222608566284,
+      "rewards/margins": 4.625195503234863,
+      "rewards/rejected": -1.772070288658142,
+      "step": 1210
+    },
+    {
+      "epoch": 2.5527348861554566,
+      "grad_norm": 6.8470663382913886,
+      "learning_rate": 6.698729810778064e-08,
+      "logits/chosen": -3.634960889816284,
+      "logits/rejected": -3.778515577316284,
+      "logps/chosen": -316.29998779296875,
+      "logps/rejected": -279.5249938964844,
+      "loss": 0.0805,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 2.7891602516174316,
+      "rewards/margins": 4.451171875,
+      "rewards/rejected": -1.6630370616912842,
+      "step": 1220
+    },
+    {
+      "epoch": 2.5736718136613455,
+      "grad_norm": 8.099289710858924,
+      "learning_rate": 6.102794338133193e-08,
+      "logits/chosen": -3.61328125,
+      "logits/rejected": -3.695507764816284,
+      "logps/chosen": -328.26251220703125,
+      "logps/rejected": -310.3125,
+      "loss": 0.0904,
+      "rewards/accuracies": 0.977343738079071,
+      "rewards/chosen": 2.910107374191284,
+      "rewards/margins": 4.730078220367432,
+      "rewards/rejected": -1.8190429210662842,
+      "step": 1230
+    },
+    {
+      "epoch": 2.594608741167234,
+      "grad_norm": 6.2645151263272965,
+      "learning_rate": 5.532892555874058e-08,
+      "logits/chosen": -3.693359375,
+      "logits/rejected": -3.7154297828674316,
+      "logps/chosen": -327.375,
+      "logps/rejected": -308.4750061035156,
+      "loss": 0.0901,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": 2.7608399391174316,
+      "rewards/margins": 4.721093654632568,
+      "rewards/rejected": -1.960205078125,
+      "step": 1240
+    },
+    {
+      "epoch": 2.615545668673122,
+      "grad_norm": 5.958871816572796,
+      "learning_rate": 4.989362450128132e-08,
+      "logits/chosen": -3.6333985328674316,
+      "logits/rejected": -3.754101514816284,
+      "logps/chosen": -328.3999938964844,
+      "logps/rejected": -299.54998779296875,
+      "loss": 0.081,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 2.850878953933716,
+      "rewards/margins": 4.644335746765137,
+      "rewards/rejected": -1.7939941883087158,
+      "step": 1250
+    },
+    {
+      "epoch": 2.6364825961790106,
+      "grad_norm": 7.218821689807589,
+      "learning_rate": 4.47252636702829e-08,
+      "logits/chosen": -3.693359375,
+      "logits/rejected": -3.789843797683716,
+      "logps/chosen": -323.95001220703125,
+      "logps/rejected": -294.7875061035156,
+      "loss": 0.0789,
+      "rewards/accuracies": 0.9828125238418579,
+      "rewards/chosen": 2.8125977516174316,
+      "rewards/margins": 4.7548828125,
+      "rewards/rejected": -1.9419434070587158,
+      "step": 1260
+    },
+    {
+      "epoch": 2.657419523684899,
+      "grad_norm": 7.334405532484607,
+      "learning_rate": 3.982690821542034e-08,
+      "logits/chosen": -3.6781249046325684,
+      "logits/rejected": -3.755078077316284,
+      "logps/chosen": -340.4125061035156,
+      "logps/rejected": -305.0375061035156,
+      "loss": 0.0803,
+      "rewards/accuracies": 0.9781249761581421,
+      "rewards/chosen": 2.909472703933716,
+      "rewards/margins": 4.7119140625,
+      "rewards/rejected": -1.8018066883087158,
+      "step": 1270
+    },
+    {
+      "epoch": 2.678356451190788,
+      "grad_norm": 6.6083908697831575,
+      "learning_rate": 3.520146315689693e-08,
+      "logits/chosen": -3.639453172683716,
+      "logits/rejected": -3.733203172683716,
+      "logps/chosen": -315.5249938964844,
+      "logps/rejected": -293.625,
+      "loss": 0.0794,
+      "rewards/accuracies": 0.98046875,
+      "rewards/chosen": 2.8868165016174316,
+      "rewards/margins": 5.292382717132568,
+      "rewards/rejected": -2.404247999191284,
+      "step": 1280
+    },
+    {
+      "epoch": 2.699293378696676,
+      "grad_norm": 6.5340040148244,
+      "learning_rate": 3.085167166259162e-08,
+      "logits/chosen": -3.684765577316284,
+      "logits/rejected": -3.807421922683716,
+      "logps/chosen": -336.4624938964844,
+      "logps/rejected": -305.7875061035156,
+      "loss": 0.0826,
+      "rewards/accuracies": 0.9765625,
+      "rewards/chosen": 2.887500047683716,
+      "rewards/margins": 4.737695217132568,
+      "rewards/rejected": -1.8518555164337158,
+      "step": 1290
+    },
+    {
+      "epoch": 2.7202303062025646,
+      "grad_norm": 7.6473031712972785,
+      "learning_rate": 2.6780113421195294e-08,
+      "logits/chosen": -3.663867235183716,
+      "logits/rejected": -3.724804639816284,
+      "logps/chosen": -328.3125,
+      "logps/rejected": -304.6499938964844,
+      "loss": 0.081,
+      "rewards/accuracies": 0.9789062738418579,
+      "rewards/chosen": 2.9146485328674316,
+      "rewards/margins": 4.792382717132568,
+      "rewards/rejected": -1.8764159679412842,
+      "step": 1300
+    },
+    {
+      "epoch": 2.7202303062025646,
+      "eval_logits/chosen": -3.675347328186035,
+      "eval_logits/rejected": -3.734375,
+      "eval_logps/chosen": -321.19049072265625,
+      "eval_logps/rejected": -301.6349182128906,
+      "eval_loss": 0.6274519562721252,
+      "eval_rewards/accuracies": 0.6696428656578064,
+      "eval_rewards/chosen": 1.1369668245315552,
+      "eval_rewards/margins": 1.3392237424850464,
+      "eval_rewards/rejected": -0.2025979608297348,
+      "eval_runtime": 37.4064,
+      "eval_samples_per_second": 53.467,
+      "eval_steps_per_second": 1.684,
+      "step": 1300
+    },
+    {
+      "epoch": 2.7411672337084534,
+      "grad_norm": 8.767488186680737,
+      "learning_rate": 2.298920311229968e-08,
+      "logits/chosen": -3.629101514816284,
+      "logits/rejected": -3.740039110183716,
+      "logps/chosen": -327.4375,
+      "logps/rejected": -300.5,
+      "loss": 0.0904,
+      "rewards/accuracies": 0.965624988079071,
+      "rewards/chosen": 2.8056640625,
+      "rewards/margins": 4.534375190734863,
+      "rewards/rejected": -1.727685570716858,
+      "step": 1310
+    },
+    {
+      "epoch": 2.7621041612143418,
+      "grad_norm": 6.549279943298276,
+      "learning_rate": 1.9481188974346696e-08,
+      "logits/chosen": -3.6734375953674316,
+      "logits/rejected": -3.8128905296325684,
+      "logps/chosen": -329.4125061035156,
+      "logps/rejected": -291.5375061035156,
+      "loss": 0.0844,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 2.9306640625,
+      "rewards/margins": 4.685156345367432,
+      "rewards/rejected": -1.755468726158142,
+      "step": 1320
+    },
+    {
+      "epoch": 2.78304108872023,
+      "grad_norm": 10.101764444445749,
+      "learning_rate": 1.6258151471287396e-08,
+      "logits/chosen": -3.6136717796325684,
+      "logits/rejected": -3.712109327316284,
+      "logps/chosen": -327.25,
+      "logps/rejected": -300.54998779296875,
+      "loss": 0.0974,
+      "rewards/accuracies": 0.969531238079071,
+      "rewards/chosen": 2.844433546066284,
+      "rewards/margins": 4.803320407867432,
+      "rewards/rejected": -1.959130883216858,
+      "step": 1330
+    },
+    {
+      "epoch": 2.803978016226119,
+      "grad_norm": 7.023228651860427,
+      "learning_rate": 1.3322002058741678e-08,
+      "logits/chosen": -3.5894532203674316,
+      "logits/rejected": -3.722460985183716,
+      "logps/chosen": -334.3500061035156,
+      "logps/rejected": -306.86248779296875,
+      "loss": 0.0771,
+      "rewards/accuracies": 0.9820312261581421,
+      "rewards/chosen": 2.909472703933716,
+      "rewards/margins": 4.759179592132568,
+      "rewards/rejected": -1.848486304283142,
+      "step": 1340
+    },
+    {
+      "epoch": 2.8249149437320074,
+      "grad_norm": 5.700111694870896,
+      "learning_rate": 1.0674482050389456e-08,
+      "logits/chosen": -3.661328077316284,
+      "logits/rejected": -3.750195264816284,
+      "logps/chosen": -325.79998779296875,
+      "logps/rejected": -303.5375061035156,
+      "loss": 0.0831,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 2.814746141433716,
+      "rewards/margins": 4.586133003234863,
+      "rewards/rejected": -1.7715332508087158,
+      "step": 1350
+    },
+    {
+      "epoch": 2.8458518712378957,
+      "grad_norm": 6.676122840652156,
+      "learning_rate": 8.317161585266963e-09,
+      "logits/chosen": -3.669140577316284,
+      "logits/rejected": -3.740429639816284,
+      "logps/chosen": -331.7875061035156,
+      "logps/rejected": -302.375,
+      "loss": 0.0785,
+      "rewards/accuracies": 0.979687511920929,
+      "rewards/chosen": 2.8916993141174316,
+      "rewards/margins": 4.768945217132568,
+      "rewards/rejected": -1.878173828125,
+      "step": 1360
+    },
+    {
+      "epoch": 2.8667887987437846,
+      "grad_norm": 11.586797872938693,
+      "learning_rate": 6.251438696579292e-09,
+      "logits/chosen": -3.6919922828674316,
+      "logits/rejected": -3.7734375,
+      "logps/chosen": -318.0625,
+      "logps/rejected": -288.7124938964844,
+      "loss": 0.0863,
+      "rewards/accuracies": 0.9765625,
+      "rewards/chosen": 2.82666015625,
+      "rewards/margins": 4.5126953125,
+      "rewards/rejected": -1.687402367591858,
+      "step": 1370
+    },
+    {
+      "epoch": 2.887725726249673,
+      "grad_norm": 7.0397468586812435,
+      "learning_rate": 4.478538482582617e-09,
+      "logits/chosen": -3.666015625,
+      "logits/rejected": -3.7943358421325684,
+      "logps/chosen": -333.29998779296875,
+      "logps/rejected": -295.7749938964844,
+      "loss": 0.0833,
+      "rewards/accuracies": 0.97265625,
+      "rewards/chosen": 2.8379883766174316,
+      "rewards/margins": 4.6533203125,
+      "rewards/rejected": -1.815527319908142,
+      "step": 1380
+    },
+    {
+      "epoch": 2.9086626537555613,
+      "grad_norm": 7.135969852937193,
+      "learning_rate": 2.999512380027047e-09,
+      "logits/chosen": -3.700976610183716,
+      "logits/rejected": -3.76171875,
+      "logps/chosen": -323.1875,
+      "logps/rejected": -299.6875,
+      "loss": 0.079,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 2.889843702316284,
+      "rewards/margins": 4.861132621765137,
+      "rewards/rejected": -1.9716796875,
+      "step": 1390
+    },
+    {
+      "epoch": 2.92959958126145,
+      "grad_norm": 7.533013184759802,
+      "learning_rate": 1.8152375405909304e-09,
+      "logits/chosen": -3.580859422683716,
+      "logits/rejected": -3.7210936546325684,
+      "logps/chosen": -323.36248779296875,
+      "logps/rejected": -296.5625,
+      "loss": 0.0772,
+      "rewards/accuracies": 0.985156238079071,
+      "rewards/chosen": 2.81591796875,
+      "rewards/margins": 4.527734279632568,
+      "rewards/rejected": -1.714111328125,
+      "step": 1400
+    },
+    {
+      "epoch": 2.92959958126145,
+      "eval_logits/chosen": -3.674603223800659,
+      "eval_logits/rejected": -3.7333829402923584,
+      "eval_logps/chosen": -321.22222900390625,
+      "eval_logps/rejected": -301.6349182128906,
+      "eval_loss": 0.6252970099449158,
+      "eval_rewards/accuracies": 0.663690447807312,
+      "eval_rewards/chosen": 1.1291543245315552,
+      "eval_rewards/margins": 1.3253967761993408,
+      "eval_rewards/rejected": -0.196025550365448,
+      "eval_runtime": 37.5622,
+      "eval_samples_per_second": 53.245,
+      "eval_steps_per_second": 1.677,
+      "step": 1400
+    },
+    {
+      "epoch": 2.9505365087673385,
+      "grad_norm": 6.693709587061416,
+      "learning_rate": 9.264163106774137e-10,
+      "logits/chosen": -3.633593797683716,
+      "logits/rejected": -3.689453125,
+      "logps/chosen": -326.8999938964844,
+      "logps/rejected": -304.3500061035156,
+      "loss": 0.0783,
+      "rewards/accuracies": 0.9828125238418579,
+      "rewards/chosen": 2.77783203125,
+      "rewards/margins": 4.6884765625,
+      "rewards/rejected": -1.9111328125,
+      "step": 1410
+    },
+    {
+      "epoch": 2.971473436273227,
+      "grad_norm": 7.492188447685543,
+      "learning_rate": 3.335758148803047e-10,
+      "logits/chosen": -3.625,
+      "logits/rejected": -3.679492235183716,
+      "logps/chosen": -323.5,
+      "logps/rejected": -305.6000061035156,
+      "loss": 0.0784,
+      "rewards/accuracies": 0.9828125238418579,
+      "rewards/chosen": 2.899218797683716,
+      "rewards/margins": 4.735937595367432,
+      "rewards/rejected": -1.835839867591858,
+      "step": 1420
+    },
+    {
+      "epoch": 2.9924103637791153,
+      "grad_norm": 8.287144286489747,
+      "learning_rate": 3.70676433677497e-11,
+      "logits/chosen": -3.6162109375,
+      "logits/rejected": -3.6527342796325684,
+      "logps/chosen": -312.9375,
+      "logps/rejected": -306.95001220703125,
+      "loss": 0.0845,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 2.7562499046325684,
+      "rewards/margins": 4.766015529632568,
+      "rewards/rejected": -2.008593797683716,
+      "step": 1430
+    },
+    {
+      "epoch": 3.0,
+      "step": 1434,
+      "total_flos": 0.0,
+      "train_loss": 0.3160468312902597,
+      "train_runtime": 10869.2749,
+      "train_samples_per_second": 16.874,
+      "train_steps_per_second": 0.132
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1434,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

nemotron_fineinstructions_1T_judged_exp_chat_7B_pf1024_DPO/hf/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d325fe7dfd0b3ebc15aaa6b54f555f1ddcc6b37aa35dc6b02619a2f32fe50f29
+size 7928