llama2-7b-dpo-lora-20231129-52 / trainer_state.json
xz-huggingface-0's picture
Model save
1e02932
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.9992254066615027,
"eval_steps": 100,
"global_step": 363,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.01,
"learning_rate": 1.3513513513513514e-08,
"logits/chosen": -0.31845107674598694,
"logits/rejected": -0.44068431854248047,
"logps/chosen": -323.8196716308594,
"logps/rejected": -252.17710876464844,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.08,
"learning_rate": 1.3513513513513515e-07,
"logits/chosen": -0.302200585603714,
"logits/rejected": -0.5121104717254639,
"logps/chosen": -364.5729675292969,
"logps/rejected": -273.8368225097656,
"loss": 0.6936,
"rewards/accuracies": 0.4340277910232544,
"rewards/chosen": -9.57948068389669e-05,
"rewards/margins": -0.0037665751297026873,
"rewards/rejected": 0.0036707809194922447,
"step": 10
},
{
"epoch": 0.17,
"learning_rate": 2.702702702702703e-07,
"logits/chosen": -0.3360111713409424,
"logits/rejected": -0.5203725695610046,
"logps/chosen": -336.6324462890625,
"logps/rejected": -258.967041015625,
"loss": 0.6941,
"rewards/accuracies": 0.5078125,
"rewards/chosen": -0.0037024090997874737,
"rewards/margins": -0.0038348871748894453,
"rewards/rejected": 0.00013247792958281934,
"step": 20
},
{
"epoch": 0.25,
"learning_rate": 4.054054054054054e-07,
"logits/chosen": -0.3064408600330353,
"logits/rejected": -0.4842294752597809,
"logps/chosen": -340.22027587890625,
"logps/rejected": -277.2881164550781,
"loss": 0.6948,
"rewards/accuracies": 0.48906248807907104,
"rewards/chosen": -0.002556417603045702,
"rewards/margins": -0.004512672312557697,
"rewards/rejected": 0.001956254942342639,
"step": 30
},
{
"epoch": 0.33,
"learning_rate": 4.95398773006135e-07,
"logits/chosen": -0.35005831718444824,
"logits/rejected": -0.4816194176673889,
"logps/chosen": -325.50445556640625,
"logps/rejected": -281.02911376953125,
"loss": 0.6941,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0014510845066979527,
"rewards/margins": -9.724525443743914e-05,
"rewards/rejected": 0.0015483299503102899,
"step": 40
},
{
"epoch": 0.41,
"learning_rate": 4.800613496932515e-07,
"logits/chosen": -0.2981815040111542,
"logits/rejected": -0.4895191192626953,
"logps/chosen": -335.34771728515625,
"logps/rejected": -276.53887939453125,
"loss": 0.6924,
"rewards/accuracies": 0.48906248807907104,
"rewards/chosen": 0.0004155976348556578,
"rewards/margins": -0.0016064945375546813,
"rewards/rejected": 0.0020220924634486437,
"step": 50
},
{
"epoch": 0.5,
"learning_rate": 4.647239263803681e-07,
"logits/chosen": -0.3579251766204834,
"logits/rejected": -0.5079066753387451,
"logps/chosen": -317.59796142578125,
"logps/rejected": -272.92578125,
"loss": 0.693,
"rewards/accuracies": 0.4671874940395355,
"rewards/chosen": 0.0026699595618993044,
"rewards/margins": 0.0002542927977629006,
"rewards/rejected": 0.0024156670551747084,
"step": 60
},
{
"epoch": 0.58,
"learning_rate": 4.4938650306748465e-07,
"logits/chosen": -0.3070736527442932,
"logits/rejected": -0.4779154360294342,
"logps/chosen": -332.0591125488281,
"logps/rejected": -271.3045349121094,
"loss": 0.6921,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": 0.004337216727435589,
"rewards/margins": 0.0024431718047708273,
"rewards/rejected": 0.0018940456211566925,
"step": 70
},
{
"epoch": 0.66,
"learning_rate": 4.340490797546012e-07,
"logits/chosen": -0.3580869734287262,
"logits/rejected": -0.46351853013038635,
"logps/chosen": -353.45233154296875,
"logps/rejected": -296.345458984375,
"loss": 0.6921,
"rewards/accuracies": 0.546875,
"rewards/chosen": 0.0069597200490534306,
"rewards/margins": 0.007152143865823746,
"rewards/rejected": -0.00019242384587414563,
"step": 80
},
{
"epoch": 0.74,
"learning_rate": 4.187116564417178e-07,
"logits/chosen": -0.36386117339134216,
"logits/rejected": -0.4859962463378906,
"logps/chosen": -336.9693298339844,
"logps/rejected": -293.84210205078125,
"loss": 0.6919,
"rewards/accuracies": 0.49531251192092896,
"rewards/chosen": 0.004757956136018038,
"rewards/margins": 0.002565748291090131,
"rewards/rejected": 0.0021922076120972633,
"step": 90
},
{
"epoch": 0.83,
"learning_rate": 4.0337423312883434e-07,
"logits/chosen": -0.3670474886894226,
"logits/rejected": -0.4790953993797302,
"logps/chosen": -318.0276184082031,
"logps/rejected": -269.90509033203125,
"loss": 0.6915,
"rewards/accuracies": 0.5296875238418579,
"rewards/chosen": 0.006049972493201494,
"rewards/margins": 0.007110539823770523,
"rewards/rejected": -0.0010605681454762816,
"step": 100
},
{
"epoch": 0.91,
"learning_rate": 3.880368098159509e-07,
"logits/chosen": -0.3364580273628235,
"logits/rejected": -0.4736698269844055,
"logps/chosen": -353.4320373535156,
"logps/rejected": -288.7911682128906,
"loss": 0.6919,
"rewards/accuracies": 0.515625,
"rewards/chosen": 0.007045713718980551,
"rewards/margins": 0.0025477451272308826,
"rewards/rejected": 0.004497968591749668,
"step": 110
},
{
"epoch": 0.99,
"learning_rate": 3.7269938650306747e-07,
"logits/chosen": -0.3278483748435974,
"logits/rejected": -0.46626392006874084,
"logps/chosen": -333.48345947265625,
"logps/rejected": -268.19891357421875,
"loss": 0.6911,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.013859955593943596,
"rewards/margins": 0.008608984760940075,
"rewards/rejected": 0.005250970367342234,
"step": 120
},
{
"epoch": 1.0,
"eval_logits/chosen": -0.17400015890598297,
"eval_logits/rejected": -0.3206101655960083,
"eval_logps/chosen": -360.1741638183594,
"eval_logps/rejected": -288.7494201660156,
"eval_loss": 0.6917624473571777,
"eval_rewards/accuracies": 0.523809552192688,
"eval_rewards/chosen": 0.00843015220016241,
"eval_rewards/margins": 9.910581866279244e-05,
"eval_rewards/rejected": 0.00833104643970728,
"eval_runtime": 153.5835,
"eval_samples_per_second": 13.022,
"eval_steps_per_second": 0.41,
"step": 121
},
{
"epoch": 1.07,
"learning_rate": 3.5736196319018404e-07,
"logits/chosen": -0.3683183193206787,
"logits/rejected": -0.46447715163230896,
"logps/chosen": -308.1628112792969,
"logps/rejected": -267.5847473144531,
"loss": 0.6904,
"rewards/accuracies": 0.5234375,
"rewards/chosen": 0.00890444777905941,
"rewards/margins": 0.004880917724221945,
"rewards/rejected": 0.004023530520498753,
"step": 130
},
{
"epoch": 1.16,
"learning_rate": 3.420245398773006e-07,
"logits/chosen": -0.30454307794570923,
"logits/rejected": -0.46795058250427246,
"logps/chosen": -331.2644958496094,
"logps/rejected": -280.22296142578125,
"loss": 0.6916,
"rewards/accuracies": 0.4921875,
"rewards/chosen": 0.006506680510938168,
"rewards/margins": 0.00031162233790382743,
"rewards/rejected": 0.006195057649165392,
"step": 140
},
{
"epoch": 1.24,
"learning_rate": 3.2668711656441716e-07,
"logits/chosen": -0.33478254079818726,
"logits/rejected": -0.490710973739624,
"logps/chosen": -340.73388671875,
"logps/rejected": -282.6800231933594,
"loss": 0.6907,
"rewards/accuracies": 0.526562511920929,
"rewards/chosen": 0.010361293330788612,
"rewards/margins": 0.005450070835649967,
"rewards/rejected": 0.004911222029477358,
"step": 150
},
{
"epoch": 1.32,
"learning_rate": 3.1134969325153373e-07,
"logits/chosen": -0.33495697379112244,
"logits/rejected": -0.4735621511936188,
"logps/chosen": -333.8702087402344,
"logps/rejected": -291.328857421875,
"loss": 0.6908,
"rewards/accuracies": 0.526562511920929,
"rewards/chosen": 0.00947122648358345,
"rewards/margins": 0.005193987395614386,
"rewards/rejected": 0.00427723815664649,
"step": 160
},
{
"epoch": 1.4,
"learning_rate": 2.960122699386503e-07,
"logits/chosen": -0.39891308546066284,
"logits/rejected": -0.5023314356803894,
"logps/chosen": -315.42413330078125,
"logps/rejected": -271.0465087890625,
"loss": 0.6908,
"rewards/accuracies": 0.5453125238418579,
"rewards/chosen": 0.015784040093421936,
"rewards/margins": 0.00708336615934968,
"rewards/rejected": 0.008700672537088394,
"step": 170
},
{
"epoch": 1.49,
"learning_rate": 2.8067484662576686e-07,
"logits/chosen": -0.3315003514289856,
"logits/rejected": -0.5215111970901489,
"logps/chosen": -342.9213562011719,
"logps/rejected": -273.6678771972656,
"loss": 0.6896,
"rewards/accuracies": 0.5609375238418579,
"rewards/chosen": 0.012953895144164562,
"rewards/margins": 0.007812326308339834,
"rewards/rejected": 0.00514157023280859,
"step": 180
},
{
"epoch": 1.57,
"learning_rate": 2.653374233128834e-07,
"logits/chosen": -0.3605652451515198,
"logits/rejected": -0.4704815447330475,
"logps/chosen": -338.06170654296875,
"logps/rejected": -281.21978759765625,
"loss": 0.6903,
"rewards/accuracies": 0.5406249761581421,
"rewards/chosen": 0.013570478186011314,
"rewards/margins": 0.008447532542049885,
"rewards/rejected": 0.005122947506606579,
"step": 190
},
{
"epoch": 1.65,
"learning_rate": 2.5e-07,
"logits/chosen": -0.2710495591163635,
"logits/rejected": -0.5266474485397339,
"logps/chosen": -361.9822082519531,
"logps/rejected": -279.62762451171875,
"loss": 0.6896,
"rewards/accuracies": 0.5296875238418579,
"rewards/chosen": 0.01437899935990572,
"rewards/margins": 0.008919144049286842,
"rewards/rejected": 0.005459855310618877,
"step": 200
},
{
"epoch": 1.74,
"learning_rate": 2.3466257668711655e-07,
"logits/chosen": -0.3298605680465698,
"logits/rejected": -0.46354150772094727,
"logps/chosen": -347.18115234375,
"logps/rejected": -278.47308349609375,
"loss": 0.6887,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.01877588778734207,
"rewards/margins": 0.012138876132667065,
"rewards/rejected": 0.006637011654675007,
"step": 210
},
{
"epoch": 1.82,
"learning_rate": 2.1932515337423312e-07,
"logits/chosen": -0.32016560435295105,
"logits/rejected": -0.4757622182369232,
"logps/chosen": -337.0078430175781,
"logps/rejected": -275.7349853515625,
"loss": 0.6878,
"rewards/accuracies": 0.5609375238418579,
"rewards/chosen": 0.018258260563015938,
"rewards/margins": 0.012606384232640266,
"rewards/rejected": 0.005651875864714384,
"step": 220
},
{
"epoch": 1.9,
"learning_rate": 2.0398773006134968e-07,
"logits/chosen": -0.3413476049900055,
"logits/rejected": -0.5039005875587463,
"logps/chosen": -342.0859069824219,
"logps/rejected": -284.2318420410156,
"loss": 0.6893,
"rewards/accuracies": 0.542187511920929,
"rewards/chosen": 0.01744121126830578,
"rewards/margins": 0.010926513001322746,
"rewards/rejected": 0.006514699663966894,
"step": 230
},
{
"epoch": 1.98,
"learning_rate": 1.8865030674846625e-07,
"logits/chosen": -0.32988137006759644,
"logits/rejected": -0.5234140753746033,
"logps/chosen": -336.7534484863281,
"logps/rejected": -276.87530517578125,
"loss": 0.6892,
"rewards/accuracies": 0.546875,
"rewards/chosen": 0.017687207087874413,
"rewards/margins": 0.010103432461619377,
"rewards/rejected": 0.007583774626255035,
"step": 240
},
{
"epoch": 2.0,
"eval_logits/chosen": -0.17397044599056244,
"eval_logits/rejected": -0.32023707032203674,
"eval_logps/chosen": -360.0880126953125,
"eval_logps/rejected": -288.69415283203125,
"eval_loss": 0.6887810230255127,
"eval_rewards/accuracies": 0.4841269850730896,
"eval_rewards/chosen": 0.017046814784407616,
"eval_rewards/margins": 0.003192761680111289,
"eval_rewards/rejected": 0.013854055665433407,
"eval_runtime": 153.4719,
"eval_samples_per_second": 13.032,
"eval_steps_per_second": 0.41,
"step": 242
},
{
"epoch": 2.07,
"learning_rate": 1.733128834355828e-07,
"logits/chosen": -0.3385268449783325,
"logits/rejected": -0.4939505457878113,
"logps/chosen": -329.9046325683594,
"logps/rejected": -284.778564453125,
"loss": 0.6887,
"rewards/accuracies": 0.551562488079071,
"rewards/chosen": 0.01729011908173561,
"rewards/margins": 0.01184510625898838,
"rewards/rejected": 0.005445011891424656,
"step": 250
},
{
"epoch": 2.15,
"learning_rate": 1.5797546012269938e-07,
"logits/chosen": -0.33671998977661133,
"logits/rejected": -0.508629560470581,
"logps/chosen": -355.34552001953125,
"logps/rejected": -275.93707275390625,
"loss": 0.6878,
"rewards/accuracies": 0.573437511920929,
"rewards/chosen": 0.020898083224892616,
"rewards/margins": 0.01348197739571333,
"rewards/rejected": 0.007416104432195425,
"step": 260
},
{
"epoch": 2.23,
"learning_rate": 1.4263803680981594e-07,
"logits/chosen": -0.3604690730571747,
"logits/rejected": -0.5304311513900757,
"logps/chosen": -337.6517028808594,
"logps/rejected": -281.8470153808594,
"loss": 0.6884,
"rewards/accuracies": 0.589062511920929,
"rewards/chosen": 0.022820422425866127,
"rewards/margins": 0.01537506002932787,
"rewards/rejected": 0.007445361465215683,
"step": 270
},
{
"epoch": 2.31,
"learning_rate": 1.273006134969325e-07,
"logits/chosen": -0.2687986493110657,
"logits/rejected": -0.4310833811759949,
"logps/chosen": -344.9039611816406,
"logps/rejected": -284.9712219238281,
"loss": 0.6877,
"rewards/accuracies": 0.546875,
"rewards/chosen": 0.021832015365362167,
"rewards/margins": 0.012980902567505836,
"rewards/rejected": 0.008851113729178905,
"step": 280
},
{
"epoch": 2.4,
"learning_rate": 1.1196319018404908e-07,
"logits/chosen": -0.3261744976043701,
"logits/rejected": -0.4705813527107239,
"logps/chosen": -340.5668640136719,
"logps/rejected": -278.6114807128906,
"loss": 0.6879,
"rewards/accuracies": 0.5546875,
"rewards/chosen": 0.021019428968429565,
"rewards/margins": 0.015442739240825176,
"rewards/rejected": 0.0055766901932656765,
"step": 290
},
{
"epoch": 2.48,
"learning_rate": 9.662576687116564e-08,
"logits/chosen": -0.32587963342666626,
"logits/rejected": -0.49200439453125,
"logps/chosen": -323.4505310058594,
"logps/rejected": -268.6414489746094,
"loss": 0.6875,
"rewards/accuracies": 0.535937488079071,
"rewards/chosen": 0.019001813605427742,
"rewards/margins": 0.00931471399962902,
"rewards/rejected": 0.009687098674476147,
"step": 300
},
{
"epoch": 2.56,
"learning_rate": 8.12883435582822e-08,
"logits/chosen": -0.2994612455368042,
"logits/rejected": -0.5041602849960327,
"logps/chosen": -336.88922119140625,
"logps/rejected": -273.3607482910156,
"loss": 0.6877,
"rewards/accuracies": 0.5609375238418579,
"rewards/chosen": 0.02191544696688652,
"rewards/margins": 0.014019708149135113,
"rewards/rejected": 0.007895738817751408,
"step": 310
},
{
"epoch": 2.64,
"learning_rate": 6.595092024539877e-08,
"logits/chosen": -0.32822033762931824,
"logits/rejected": -0.48569512367248535,
"logps/chosen": -316.4683532714844,
"logps/rejected": -280.8128356933594,
"loss": 0.6874,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": 0.020111314952373505,
"rewards/margins": 0.01264654565602541,
"rewards/rejected": 0.0074647702276706696,
"step": 320
},
{
"epoch": 2.73,
"learning_rate": 5.061349693251534e-08,
"logits/chosen": -0.2966347634792328,
"logits/rejected": -0.4929285943508148,
"logps/chosen": -344.79669189453125,
"logps/rejected": -279.22320556640625,
"loss": 0.6867,
"rewards/accuracies": 0.5531250238418579,
"rewards/chosen": 0.022619571536779404,
"rewards/margins": 0.016952764242887497,
"rewards/rejected": 0.0056668114848434925,
"step": 330
},
{
"epoch": 2.81,
"learning_rate": 3.5276073619631896e-08,
"logits/chosen": -0.31011176109313965,
"logits/rejected": -0.46126970648765564,
"logps/chosen": -333.1022033691406,
"logps/rejected": -285.73016357421875,
"loss": 0.6883,
"rewards/accuracies": 0.559374988079071,
"rewards/chosen": 0.021285008639097214,
"rewards/margins": 0.011302657425403595,
"rewards/rejected": 0.009982350282371044,
"step": 340
},
{
"epoch": 2.89,
"learning_rate": 1.9938650306748464e-08,
"logits/chosen": -0.34948185086250305,
"logits/rejected": -0.4592677056789398,
"logps/chosen": -327.64312744140625,
"logps/rejected": -290.9291076660156,
"loss": 0.6869,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": 0.02595512941479683,
"rewards/margins": 0.016743769869208336,
"rewards/rejected": 0.009211359545588493,
"step": 350
},
{
"epoch": 2.97,
"learning_rate": 4.601226993865031e-09,
"logits/chosen": -0.33892008662223816,
"logits/rejected": -0.4774000644683838,
"logps/chosen": -340.5291748046875,
"logps/rejected": -276.88043212890625,
"loss": 0.6867,
"rewards/accuracies": 0.5625,
"rewards/chosen": 0.024597328156232834,
"rewards/margins": 0.015405513346195221,
"rewards/rejected": 0.009191813878715038,
"step": 360
},
{
"epoch": 3.0,
"eval_logits/chosen": -0.1736111342906952,
"eval_logits/rejected": -0.31976231932640076,
"eval_logps/chosen": -359.9693298339844,
"eval_logps/rejected": -288.6960144042969,
"eval_loss": 0.6869024038314819,
"eval_rewards/accuracies": 0.567460298538208,
"eval_rewards/chosen": 0.02891545742750168,
"eval_rewards/margins": 0.01524704322218895,
"eval_rewards/rejected": 0.013668415136635303,
"eval_runtime": 152.857,
"eval_samples_per_second": 13.084,
"eval_steps_per_second": 0.412,
"step": 363
},
{
"epoch": 3.0,
"step": 363,
"total_flos": 0.0,
"train_loss": 0.6900519354284302,
"train_runtime": 24510.12,
"train_samples_per_second": 7.585,
"train_steps_per_second": 0.015
}
],
"logging_steps": 10,
"max_steps": 363,
"num_train_epochs": 3,
"save_steps": 500,
"total_flos": 0.0,
"trial_name": null,
"trial_params": null
}