Model save

README.md

@@ -0,0 +1,65 @@
+---
+license: gemma
+base_model: google/gemma-2-2b-it
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: gemma-2-2b-it-mt-sft-full_new
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma-2-2b-it-mt-sft-full_new
+
+This model is a fine-tuned version of [google/gemma-2-2b-it](https://huggingface.co/google/gemma-2-2b-it) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.8862
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-05
+- train_batch_size: 1
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.7473        | 0.9982 | 283  | 1.8862          |
+
+
+### Framework versions
+
+- Transformers 4.43.3
+- Pytorch 2.1.2
+- Datasets 2.20.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9982363315696648,
+    "total_flos": 1.1264430584732058e+17,
+    "train_loss": 1.8635691063142918,
+    "train_runtime": 3704.3906,
+    "train_samples": 117404,
+    "train_samples_per_second": 1.224,
+    "train_steps_per_second": 0.076
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "eos_token_id": [
+    1,
+    107
+  ],
+  "pad_token_id": 0,
+  "transformers_version": "4.43.3"
+}

runs/Aug02_00-26-24_poseidon/events.out.tfevents.1722558421.poseidon.1257643.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bdfe67970473e8298de7099f94843c627cfe3e26f158379b9a4116aa6b1ba6d
-size 11354
+oid sha256:cfb083a22e6752d00112451715fdc12466e4fd8b8265d59092c0f16e02bb4306
+size 11979

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9982363315696648,
+    "total_flos": 1.1264430584732058e+17,
+    "train_loss": 1.8635691063142918,
+    "train_runtime": 3704.3906,
+    "train_samples": 117404,
+    "train_samples_per_second": 1.224,
+    "train_steps_per_second": 0.076
+}

trainer_state.json

@@ -0,0 +1,253 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9982363315696648,
+  "eval_steps": 500,
+  "global_step": 283,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003527336860670194,
+      "grad_norm": 45.75,
+      "learning_rate": 3.4482758620689656e-07,
+      "loss": 3.4754,
+      "step": 1
+    },
+    {
+      "epoch": 0.03527336860670194,
+      "grad_norm": 27.25,
+      "learning_rate": 3.448275862068966e-06,
+      "loss": 3.2945,
+      "step": 10
+    },
+    {
+      "epoch": 0.07054673721340388,
+      "grad_norm": 6.84375,
+      "learning_rate": 6.896551724137932e-06,
+      "loss": 2.4678,
+      "step": 20
+    },
+    {
+      "epoch": 0.10582010582010581,
+      "grad_norm": 3.375,
+      "learning_rate": 9.999617556940085e-06,
+      "loss": 2.0003,
+      "step": 30
+    },
+    {
+      "epoch": 0.14109347442680775,
+      "grad_norm": 2.828125,
+      "learning_rate": 9.953795138672291e-06,
+      "loss": 1.8951,
+      "step": 40
+    },
+    {
+      "epoch": 0.1763668430335097,
+      "grad_norm": 2.765625,
+      "learning_rate": 9.832286532071802e-06,
+      "loss": 1.8598,
+      "step": 50
+    },
+    {
+      "epoch": 0.21164021164021163,
+      "grad_norm": 2.765625,
+      "learning_rate": 9.636948197292051e-06,
+      "loss": 1.8314,
+      "step": 60
+    },
+    {
+      "epoch": 0.24691358024691357,
+      "grad_norm": 2.75,
+      "learning_rate": 9.370764596463764e-06,
+      "loss": 1.8099,
+      "step": 70
+    },
+    {
+      "epoch": 0.2821869488536155,
+      "grad_norm": 2.6875,
+      "learning_rate": 9.037802595813621e-06,
+      "loss": 1.8067,
+      "step": 80
+    },
+    {
+      "epoch": 0.31746031746031744,
+      "grad_norm": 3.140625,
+      "learning_rate": 8.643149330352939e-06,
+      "loss": 1.8112,
+      "step": 90
+    },
+    {
+      "epoch": 0.3527336860670194,
+      "grad_norm": 2.75,
+      "learning_rate": 8.192834480466072e-06,
+      "loss": 1.779,
+      "step": 100
+    },
+    {
+      "epoch": 0.3880070546737213,
+      "grad_norm": 2.703125,
+      "learning_rate": 7.693738147889868e-06,
+      "loss": 1.7797,
+      "step": 110
+    },
+    {
+      "epoch": 0.42328042328042326,
+      "grad_norm": 2.640625,
+      "learning_rate": 7.153485738594111e-06,
+      "loss": 1.7604,
+      "step": 120
+    },
+    {
+      "epoch": 0.4585537918871252,
+      "grad_norm": 2.609375,
+      "learning_rate": 6.5803314585870225e-06,
+      "loss": 1.777,
+      "step": 130
+    },
+    {
+      "epoch": 0.49382716049382713,
+      "grad_norm": 2.640625,
+      "learning_rate": 5.9830322026464435e-06,
+      "loss": 1.7684,
+      "step": 140
+    },
+    {
+      "epoch": 0.5291005291005291,
+      "grad_norm": 2.625,
+      "learning_rate": 5.3707137627582315e-06,
+      "loss": 1.7621,
+      "step": 150
+    },
+    {
+      "epoch": 0.564373897707231,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.752731400386306e-06,
+      "loss": 1.754,
+      "step": 160
+    },
+    {
+      "epoch": 0.599647266313933,
+      "grad_norm": 2.71875,
+      "learning_rate": 4.138526912810418e-06,
+      "loss": 1.757,
+      "step": 170
+    },
+    {
+      "epoch": 0.6349206349206349,
+      "grad_norm": 2.53125,
+      "learning_rate": 3.537484377332945e-06,
+      "loss": 1.755,
+      "step": 180
+    },
+    {
+      "epoch": 0.6701940035273368,
+      "grad_norm": 2.65625,
+      "learning_rate": 2.9587867773560488e-06,
+      "loss": 1.7516,
+      "step": 190
+    },
+    {
+      "epoch": 0.7054673721340388,
+      "grad_norm": 2.5625,
+      "learning_rate": 2.4112757008569102e-06,
+      "loss": 1.7561,
+      "step": 200
+    },
+    {
+      "epoch": 0.7407407407407407,
+      "grad_norm": 2.6875,
+      "learning_rate": 1.9033162548474577e-06,
+      "loss": 1.7457,
+      "step": 210
+    },
+    {
+      "epoch": 0.7760141093474426,
+      "grad_norm": 2.625,
+      "learning_rate": 1.4426692597128339e-06,
+      "loss": 1.7584,
+      "step": 220
+    },
+    {
+      "epoch": 0.8112874779541446,
+      "grad_norm": 2.609375,
+      "learning_rate": 1.036372676097897e-06,
+      "loss": 1.7584,
+      "step": 230
+    },
+    {
+      "epoch": 0.8465608465608465,
+      "grad_norm": 2.6875,
+      "learning_rate": 6.90634075952149e-07,
+      "loss": 1.7567,
+      "step": 240
+    },
+    {
+      "epoch": 0.8818342151675485,
+      "grad_norm": 2.65625,
+      "learning_rate": 4.1073580060622455e-07,
+      "loss": 1.7539,
+      "step": 250
+    },
+    {
+      "epoch": 0.9171075837742504,
+      "grad_norm": 2.625,
+      "learning_rate": 2.0095425491515386e-07,
+      "loss": 1.7624,
+      "step": 260
+    },
+    {
+      "epoch": 0.9523809523809523,
+      "grad_norm": 2.59375,
+      "learning_rate": 6.449457052677965e-08,
+      "loss": 1.7407,
+      "step": 270
+    },
+    {
+      "epoch": 0.9876543209876543,
+      "grad_norm": 2.59375,
+      "learning_rate": 3.4416365177236675e-09,
+      "loss": 1.7473,
+      "step": 280
+    },
+    {
+      "epoch": 0.9982363315696648,
+      "eval_loss": 1.8862216472625732,
+      "eval_runtime": 61.8534,
+      "eval_samples_per_second": 5.384,
+      "eval_steps_per_second": 2.7,
+      "step": 283
+    },
+    {
+      "epoch": 0.9982363315696648,
+      "step": 283,
+      "total_flos": 1.1264430584732058e+17,
+      "train_loss": 1.8635691063142918,
+      "train_runtime": 3704.3906,
+      "train_samples_per_second": 1.224,
+      "train_steps_per_second": 0.076
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 283,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1264430584732058e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}