Upload folder using huggingface_hub

.gitattributes

@@ -37,3 +37,5 @@ llmtf_eval/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
+llmtf_eval/ruparam.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval/vikhrmodels_habr_qa_sbs.jsonl filter=lfs diff=lfs merge=lfs -text

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/kto1",
+  "_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/sft1",
   "architectures": [
     "Qwen2ForCausalLM"
   ],

llmtf_eval/NEREL_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 128,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 200,
+        "method": "generate"
+    }
+}

llmtf_eval/NEREL_total.jsonl

@@ -0,0 +1,10 @@
+{
+    "task_name": "NEREL",
+    "results": {
+        "tp": 2.0,
+        "fp": 27.0,
+        "fn": 519.0,
+        "micro-f1": 0.00727272727272595
+    },
+    "leaderboard_result": 0.00727272727272595
+}

llmtf_eval/daru_treewayabstractive_params.jsonl

@@ -1,7 +1,7 @@
 {
     "custom_generation_config": null,
     "model_params": {
-        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
         "generation_config": {
             "bos_token_id": 145109,
             "do_sample": true,
@@ -36,7 +36,7 @@
         },
         "load_in_8bit": false,
         "torch_dtype": "auto",
-        "use_flash_attention_2": true,
+        "attn_implementation": "flash_attention_2",
         "device_map": "cuda:0",
         "use_fast_tokenizer": true,
         "leading_space": false,
@@ -47,7 +47,7 @@
     "task_params": {
         "max_len": 4000,
         "few_shot_count": 0,
-        "batch_size": 2,
+        "batch_size": 16,
         "max_sample_per_dataset": 200,
         "method": "generate"
     }

llmtf_eval/daru_treewayabstractive_total.jsonl

@@ -1,8 +1,8 @@
 {
     "task_name": "daru/treewayabstractive",
     "results": {
-        "rouge1": 0.31023763628891676,
-        "rouge2": 0.09443696323171702
+        "rouge1": 0.3138417117532064,
+        "rouge2": 0.10462617373556911
     },
-    "leaderboard_result": 0.2023372997603169
+    "leaderboard_result": 0.20923394274438775
 }

llmtf_eval/darumeru_MultiQ_params.jsonl

@@ -1,7 +1,7 @@
 {
     "custom_generation_config": null,
     "model_params": {
-        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
         "generation_config": {
             "bos_token_id": 145109,
             "do_sample": true,
@@ -36,7 +36,7 @@
         },
         "load_in_8bit": false,
         "torch_dtype": "auto",
-        "use_flash_attention_2": true,
+        "attn_implementation": "flash_attention_2",
         "device_map": "cuda:0",
         "use_fast_tokenizer": true,
         "leading_space": false,
@@ -47,7 +47,7 @@
     "task_params": {
         "max_len": 4000,
         "few_shot_count": 0,
-        "batch_size": 2,
+        "batch_size": 16,
         "max_sample_per_dataset": 10000000000000,
         "method": "generate"
     }

llmtf_eval/darumeru_MultiQ_total.jsonl

@@ -1,8 +1,8 @@
 {
     "task_name": "darumeru/MultiQ",
     "results": {
-        "f1": 0.20613243758223346,
-        "em": 0.11281070745697896
+        "f1": 0.3016876852043635,
+        "em": 0.21319311663479923
     },
-    "leaderboard_result": 0.1594715725196062
+    "leaderboard_result": 0.25744040091958137
 }

llmtf_eval/darumeru_PARus.jsonl

@@ -6,8 +6,8 @@
         }
     },
     "predict": {
-        "1": 0.6791735887527466,
-        "2": 0.3208189010620117
+        "1": 0.5619974732398987,
+        "2": 0.4376840591430664
     },
     "sample": {
         "messages": [
@@ -47,8 +47,8 @@
         }
     },
     "predict": {
-        "1": 0.7772925496101379,
-        "2": 0.22269804775714874
+        "1": 0.622265100479126,
+        "2": 0.37742286920547485
     },
     "sample": {
         "messages": [
@@ -88,8 +88,8 @@
         }
     },
     "predict": {
-        "1": 0.9324424862861633,
-        "2": 0.06754591315984726
+        "1": 0.6510406732559204,
+        "2": 0.34847697615623474
     },
     "sample": {
         "messages": [
@@ -129,8 +129,8 @@
         }
     },
     "predict": {
-        "1": 0.05340291187167168,
-        "2": 0.9465891718864441
+        "1": 0.32066380977630615,
+        "2": 0.678845226764679
     },
     "sample": {
         "messages": [
@@ -170,8 +170,8 @@
         }
     },
     "predict": {
-        "1": 0.9980727434158325,
-        "2": 0.0019267337629571557
+        "1": 0.7771914601325989,
+        "2": 0.2226690798997879
     },
     "sample": {
         "messages": [
@@ -211,8 +211,8 @@
         }
     },
     "predict": {
-        "1": 1.3709549193663406e-06,
-        "2": 0.9999969005584717
+        "1": 0.04741984233260155,
+        "2": 0.9524530172348022
     },
     "sample": {
         "messages": [
@@ -252,8 +252,8 @@
         }
     },
     "predict": {
-        "1": 0.003593580098822713,
-        "2": 0.9964001178741455
+        "1": 0.24495473504066467,
+        "2": 0.7545137405395508
     },
     "sample": {
         "messages": [
@@ -293,8 +293,8 @@
         }
     },
     "predict": {
-        "1": 0.18242034316062927,
-        "2": 0.8175512552261353
+        "1": 0.4071190059185028,
+        "2": 0.5923546552658081
     },
     "sample": {
         "messages": [
@@ -334,8 +334,8 @@
         }
     },
     "predict": {
-        "1": 0.1480443924665451,
-        "2": 0.8519367575645447
+        "1": 0.46866750717163086,
+        "2": 0.5310698747634888
     },
     "sample": {
         "messages": [
@@ -375,8 +375,8 @@
         }
     },
     "predict": {
-        "1": 0.7981746196746826,
-        "2": 0.20181015133857727
+        "1": 0.6511845588684082,
+        "2": 0.34855398535728455
     },
     "sample": {
         "messages": [
@@ -416,8 +416,8 @@
         }
     },
     "predict": {
-        "1": 0.5312058329582214,
-        "2": 0.4687875211238861
+        "1": 0.5619243383407593,
+        "2": 0.4376271367073059
     },
     "sample": {
         "messages": [
@@ -457,8 +457,8 @@
         }
     },
     "predict": {
-        "1": 0.0019267323659732938,
-        "2": 0.9980720281600952
+        "1": 0.2688515782356262,
+        "2": 0.7308142781257629
     },
     "sample": {
         "messages": [
@@ -498,8 +498,8 @@
         }
     },
     "predict": {
-        "1": 0.00460954662412405,
-        "2": 0.9953848719596863
+        "1": 0.29402416944503784,
+        "2": 0.7053273320198059
     },
     "sample": {
         "messages": [
@@ -539,8 +539,8 @@
         }
     },
     "predict": {
-        "1": 0.0059110443107783794,
-        "2": 0.994084894657135
+        "1": 0.32064735889434814,
+        "2": 0.6788104176521301
     },
     "sample": {
         "messages": [
@@ -580,8 +580,8 @@
         }
     },
     "predict": {
-        "1": 0.5926629304885864,
-        "2": 0.4073309004306793
+        "1": 0.730933427810669,
+        "2": 0.2688954174518585
     },
     "sample": {
         "messages": [
@@ -621,8 +621,8 @@
         }
     },
     "predict": {
-        "1": 0.004609555937349796,
-        "2": 0.9953868985176086
+        "1": 0.4072516858577728,
+        "2": 0.5925476551055908
     },
     "sample": {
         "messages": [
@@ -662,8 +662,8 @@
         }
     },
     "predict": {
-        "1": 0.02595728635787964,
-        "2": 0.9740400314331055
+        "1": 0.37738102674484253,
+        "2": 0.6221961379051208
     },
     "sample": {
         "messages": [
@@ -703,8 +703,8 @@
         }
     },
     "predict": {
-        "1": 0.20181089639663696,
-        "2": 0.7981775999069214
+        "1": 0.3773842751979828,
+        "2": 0.6222015023231506
     },
     "sample": {
         "messages": [
@@ -739,13 +739,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 9
         }
     },
     "predict": {
-        "1": 0.18240754306316376,
-        "2": 0.8174938559532166
+        "1": 0.4990942180156708,
+        "2": 0.4990942180156708
     },
     "sample": {
         "messages": [
@@ -774,7 +774,7 @@
         "prompt_len": 78,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -785,8 +785,8 @@
         }
     },
     "predict": {
-        "1": 0.005219845101237297,
-        "2": 0.9947263598442078
+        "1": 0.2684953212738037,
+        "2": 0.7298458218574524
     },
     "sample": {
         "messages": [
@@ -821,13 +821,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 10
         }
     },
     "predict": {
-        "1": 0.4073287546634674,
-        "2": 0.5926598310470581
+        "1": 0.6222006678581238,
+        "2": 0.3773837685585022
     },
     "sample": {
         "messages": [
@@ -856,7 +856,7 @@
         "prompt_len": 87,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -867,8 +867,8 @@
         }
     },
     "predict": {
-        "1": 0.1480454057455063,
-        "2": 0.8519425392150879
+        "1": 0.46855002641677856,
+        "2": 0.5309367775917053
     },
     "sample": {
         "messages": [
@@ -908,8 +908,8 @@
         }
     },
     "predict": {
-        "1": 0.03732459619641304,
-        "2": 0.9626140594482422
+        "1": 0.320069283246994,
+        "2": 0.6775866150856018
     },
     "sample": {
         "messages": [
@@ -949,8 +949,8 @@
         }
     },
     "predict": {
-        "1": 0.26892003417015076,
-        "2": 0.7310003638267517
+        "1": 0.467572420835495,
+        "2": 0.5298289656639099
     },
     "sample": {
         "messages": [
@@ -990,8 +990,8 @@
         }
     },
     "predict": {
-        "1": 0.26893946528434753,
-        "2": 0.7310531735420227
+        "1": 0.4377047121524811,
+        "2": 0.5620239973068237
     },
     "sample": {
         "messages": [
@@ -1031,8 +1031,8 @@
         }
     },
     "predict": {
-        "1": 0.5312056541442871,
-        "2": 0.4687873125076294
+        "1": 0.5620283484458923,
+        "2": 0.4377081096172333
     },
     "sample": {
         "messages": [
@@ -1072,8 +1072,8 @@
         }
     },
     "predict": {
-        "1": 0.025957094505429268,
-        "2": 0.9740327596664429
+        "1": 0.37745413184165955,
+        "2": 0.6223166584968567
     },
     "sample": {
         "messages": [
@@ -1108,13 +1108,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 13
         }
     },
     "predict": {
-        "1": 0.2689380645751953,
-        "2": 0.7310493588447571
+        "1": 0.4998833239078522,
+        "2": 0.4998833239078522
     },
     "sample": {
         "messages": [
@@ -1143,7 +1143,7 @@
         "prompt_len": 106,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1154,8 +1154,8 @@
         }
     },
     "predict": {
-        "1": 0.09534876048564911,
-        "2": 0.9046438932418823
+        "1": 0.43762025237083435,
+        "2": 0.5619155168533325
     },
     "sample": {
         "messages": [
@@ -1195,8 +1195,8 @@
         }
     },
     "predict": {
-        "1": 0.43782007694244385,
-        "2": 0.5621721148490906
+        "1": 0.4685898721218109,
+        "2": 0.5309818983078003
     },
     "sample": {
         "messages": [
@@ -1231,13 +1231,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 15
         }
     },
     "predict": {
-        "1": 0.3208208382129669,
-        "2": 0.6791777014732361
+        "1": 0.6512307524681091,
+        "2": 0.34857872128486633
     },
     "sample": {
         "messages": [
@@ -1266,7 +1266,7 @@
         "prompt_len": 91,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1277,8 +1277,8 @@
         }
     },
     "predict": {
-        "1": 0.0013250199845060706,
-        "2": 0.9986732006072998
+        "1": 0.2688847780227661,
+        "2": 0.7309045195579529
     },
     "sample": {
         "messages": [
@@ -1313,13 +1313,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 16
         }
     },
     "predict": {
-        "1": 0.8354753851890564,
-        "2": 0.16451483964920044
+        "1": 0.4685722291469574,
+        "2": 0.5309618711471558
     },
     "sample": {
         "messages": [
@@ -1348,7 +1348,7 @@
         "prompt_len": 92,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "1"
+        "generated_token": "2"
     }
 }
 {
@@ -1359,8 +1359,8 @@
         }
     },
     "predict": {
-        "1": 0.437816858291626,
-        "2": 0.5621679425239563
+        "1": 0.3773617446422577,
+        "2": 0.6221643090248108
     },
     "sample": {
         "messages": [
@@ -1395,13 +1395,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 17
         }
     },
     "predict": {
-        "1": 0.047425612807273865,
-        "2": 0.9525689482688904
+        "1": 0.5308536887168884,
+        "2": 0.46847671270370483
     },
     "sample": {
         "messages": [
@@ -1430,7 +1430,7 @@
         "prompt_len": 75,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1441,8 +1441,8 @@
         }
     },
     "predict": {
-        "1": 0.003172677243128419,
-        "2": 0.9968255758285522
+        "1": 0.348453551530838,
+        "2": 0.6509969234466553
     },
     "sample": {
         "messages": [
@@ -1482,8 +1482,8 @@
         }
     },
     "predict": {
-        "1": 0.04741906747221947,
-        "2": 0.9524374604225159
+        "1": 0.4063173532485962,
+        "2": 0.5911882519721985
     },
     "sample": {
         "messages": [
@@ -1518,13 +1518,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 18
         }
     },
     "predict": {
-        "1": 0.22267712652683258,
-        "2": 0.7772195339202881
+        "1": 0.4990358054637909,
+        "2": 0.4990358054637909
     },
     "sample": {
         "messages": [
@@ -1553,7 +1553,7 @@
         "prompt_len": 74,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1564,8 +1564,8 @@
         }
     },
     "predict": {
-        "1": 0.9959231615066528,
-        "2": 0.004070110619068146
+        "1": 0.730846107006073,
+        "2": 0.2688632905483246
     },
     "sample": {
         "messages": [
@@ -1605,8 +1605,8 @@
         }
     },
     "predict": {
-        "1": 0.02931191585958004,
-        "2": 0.9706773161888123
+        "1": 0.4376976490020752,
+        "2": 0.5620148777961731
     },
     "sample": {
         "messages": [
@@ -1641,13 +1641,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 20
         }
     },
     "predict": {
-        "1": 0.29421061277389526,
-        "2": 0.7057746052742004
+        "1": 0.592413604259491,
+        "2": 0.4071595370769501
     },
     "sample": {
         "messages": [
@@ -1676,7 +1676,7 @@
         "prompt_len": 80,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1687,8 +1687,8 @@
         }
     },
     "predict": {
-        "1": 0.009708438068628311,
-        "2": 0.9902875423431396
+        "1": 0.3774004280567169,
+        "2": 0.6222280859947205
     },
     "sample": {
         "messages": [
@@ -1728,8 +1728,8 @@
         }
     },
     "predict": {
-        "1": 0.005220096092671156,
-        "2": 0.9947742819786072
+        "1": 0.34851840138435364,
+        "2": 0.6511180400848389
     },
     "sample": {
         "messages": [
@@ -1769,8 +1769,8 @@
         }
     },
     "predict": {
-        "1": 0.8807904720306396,
-        "2": 0.11920202523469925
+        "1": 0.7055380940437317,
+        "2": 0.29411202669143677
     },
     "sample": {
         "messages": [
@@ -1810,8 +1810,8 @@
         }
     },
     "predict": {
-        "1": 0.00029595597879961133,
-        "2": 0.9997001886367798
+        "1": 0.3207487165927887,
+        "2": 0.6790250539779663
     },
     "sample": {
         "messages": [
@@ -1846,13 +1846,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 22
         }
     },
     "predict": {
-        "1": 0.7981850504875183,
-        "2": 0.20181278884410858
+        "1": 0.4686760902404785,
+        "2": 0.5310795903205872
     },
     "sample": {
         "messages": [
@@ -1881,7 +1881,7 @@
         "prompt_len": 96,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "1"
+        "generated_token": "2"
     }
 }
 {
@@ -1892,8 +1892,8 @@
         }
     },
     "predict": {
-        "1": 0.37753984332084656,
-        "2": 0.6224579215049744
+        "1": 0.43775004148483276,
+        "2": 0.5620821714401245
     },
     "sample": {
         "messages": [
@@ -1933,8 +1933,8 @@
         }
     },
     "predict": {
-        "1": 0.003172674449160695,
-        "2": 0.9968246221542358
+        "1": 0.32076162099838257,
+        "2": 0.6790523529052734
     },
     "sample": {
         "messages": [
@@ -1969,13 +1969,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 24
         }
     },
     "predict": {
-        "1": 0.9398970007896423,
-        "2": 0.06008560210466385
+        "1": 0.37734633684158325,
+        "2": 0.6221389174461365
     },
     "sample": {
         "messages": [
@@ -2004,7 +2004,7 @@
         "prompt_len": 85,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "1"
+        "generated_token": "2"
     }
 }
 {
@@ -2015,8 +2015,8 @@
         }
     },
     "predict": {
-        "1": 0.24508117139339447,
-        "2": 0.7549031972885132
+        "1": 0.37736234068870544,
+        "2": 0.622165322303772
     },
     "sample": {
         "messages": [
@@ -2056,8 +2056,8 @@
         }
     },
     "predict": {
-        "1": 0.07585714012384415,
-        "2": 0.9241291880607605
+        "1": 0.407107949256897,
+        "2": 0.5923385620117188
     },
     "sample": {
         "messages": [
@@ -2097,8 +2097,8 @@
         }
     },
     "predict": {
-        "1": 0.8807794451713562,
-        "2": 0.11920053511857986
+        "1": 0.5619068145751953,
+        "2": 0.43761345744132996
     },
     "sample": {
         "messages": [
@@ -2138,8 +2138,8 @@
         }
     },
     "predict": {
-        "1": 0.033085692673921585,
-        "2": 0.9669057130813599
+        "1": 0.43754732608795166,
+        "2": 0.5618218779563904
     },
     "sample": {
         "messages": [
@@ -2174,13 +2174,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 26
         }
     },
     "predict": {
-        "1": 0.13296259939670563,
-        "2": 0.8670250773429871
+        "1": 0.5308565497398376,
+        "2": 0.4684792160987854
     },
     "sample": {
         "messages": [
@@ -2209,7 +2209,7 @@
         "prompt_len": 80,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -2220,8 +2220,8 @@
         }
     },
     "predict": {
-        "1": 0.0004305326147004962,
-        "2": 0.9995126724243164
+        "1": 0.3765822649002075,
+        "2": 0.6208791732788086
     },
     "sample": {
         "messages": [
@@ -2261,8 +2261,8 @@
         }
     },
     "predict": {
-        "1": 0.3208002746105194,
-        "2": 0.6791341304779053
+        "1": 0.4678250551223755,
+        "2": 0.5301152467727661
     },
     "sample": {
         "messages": [
@@ -2302,8 +2302,8 @@
         }
     },
     "predict": {
-        "1": 0.014063562266528606,
-        "2": 0.9859318733215332
+        "1": 0.4685414433479309,
+        "2": 0.5309270024299622
     },
     "sample": {
         "messages": [
@@ -2343,8 +2343,8 @@
         }
     },
     "predict": {
-        "1": 0.5926593542098999,
-        "2": 0.40732845664024353
+        "1": 0.49978330731391907,
+        "2": 0.49978330731391907
     },
     "sample": {
         "messages": [
@@ -2379,13 +2379,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 29
         }
     },
     "predict": {
-        "1": 0.4378196895122528,
-        "2": 0.5621716380119324
+        "1": 0.5925133228302002,
+        "2": 0.4072280526161194
     },
     "sample": {
         "messages": [
@@ -2414,19 +2414,19 @@
         "prompt_len": 91,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 29
         }
     },
     "predict": {
-        "1": 0.407329797744751,
-        "2": 0.5926613211631775
+        "1": 0.5924868583679199,
+        "2": 0.4072098731994629
     },
     "sample": {
         "messages": [
@@ -2455,7 +2455,7 @@
         "prompt_len": 91,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -2466,8 +2466,8 @@
         }
     },
     "predict": {
-        "1": 0.00015842854918446392,
-        "2": 0.9997932314872742
+        "1": 0.2013276070356369,
+        "2": 0.7962661385536194
     },
     "sample": {
         "messages": [
@@ -2507,8 +2507,8 @@
         }
     },
     "predict": {
-        "1": 0.07585276663303375,
-        "2": 0.9240758419036865
+        "1": 0.40639403462409973,
+        "2": 0.591299831867218
     },
     "sample": {
         "messages": [
@@ -2548,8 +2548,8 @@
         }
     },
     "predict": {
-        "1": 0.0021827055606991053,
-        "2": 0.9978122711181641
+        "1": 0.4685439169406891,
+        "2": 0.5309298038482666
     },
     "sample": {
         "messages": [
@@ -2584,13 +2584,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 31
         }
     },
     "predict": {
-        "1": 0.2689363360404968,
-        "2": 0.7310447096824646
+        "1": 0.49973681569099426,
+        "2": 0.49973681569099426
     },
     "sample": {
         "messages": [
@@ -2619,19 +2619,19 @@
         "prompt_len": 80,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 32
         }
     },
     "predict": {
-        "1": 0.0474257692694664,
-        "2": 0.9525721073150635
+        "1": 0.5311156511306763,
+        "2": 0.4687079191207886
     },
     "sample": {
         "messages": [
@@ -2660,7 +2660,7 @@
         "prompt_len": 83,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -2671,8 +2671,8 @@
         }
     },
     "predict": {
-        "1": 0.0010322295129299164,
-        "2": 0.9989663362503052
+        "1": 0.29415959119796753,
+        "2": 0.7056521773338318
     },
     "sample": {
         "messages": [
@@ -2712,8 +2712,8 @@
         }
     },
     "predict": {
-        "1": 0.01590634509921074,
-        "2": 0.9840907454490662
+        "1": 0.4071362614631653,
+        "2": 0.5923797488212585
     },
     "sample": {
         "messages": [
@@ -2753,8 +2753,8 @@
         }
     },
     "predict": {
-        "1": 0.2450830489397049,
-        "2": 0.7549089789390564
+        "1": 0.43763670325279236,
+        "2": 0.561936616897583
     },
     "sample": {
         "messages": [
@@ -2794,8 +2794,8 @@
         }
     },
     "predict": {
-        "1": 0.9525681734085083,
-        "2": 0.04742557555437088
+        "1": 0.8173764944076538,
+        "2": 0.18238134682178497
     },
     "sample": {
         "messages": [
@@ -2835,8 +2835,8 @@
         }
     },
     "predict": {
-        "1": 0.7549096941947937,
-        "2": 0.2450833022594452
+        "1": 0.5924760699272156,
+        "2": 0.40720245242118835
     },
     "sample": {
         "messages": [
@@ -2876,8 +2876,8 @@
         }
     },
     "predict": {
-        "1": 0.047425732016563416,
-        "2": 0.9525713324546814
+        "1": 0.43765684962272644,
+        "2": 0.5619625449180603
     },
     "sample": {
         "messages": [
@@ -2917,8 +2917,8 @@
         }
     },
     "predict": {
-        "1": 0.029312150552868843,
-        "2": 0.9706850647926331
+        "1": 0.29410114884376526,
+        "2": 0.7055119872093201
     },
     "sample": {
         "messages": [
@@ -2958,8 +2958,8 @@
         }
     },
     "predict": {
-        "1": 0.017984773963689804,
-        "2": 0.9819353818893433
+        "1": 0.37653520703315735,
+        "2": 0.620801568031311
     },
     "sample": {
         "messages": [
@@ -2999,8 +2999,8 @@
         }
     },
     "predict": {
-        "1": 0.017984716221690178,
-        "2": 0.9819322824478149
+        "1": 0.4062846899032593,
+        "2": 0.5911407470703125
     },
     "sample": {
         "messages": [
@@ -3040,8 +3040,8 @@
         }
     },
     "predict": {
-        "1": 0.10668963193893433,
-        "2": 0.8933013677597046
+        "1": 0.46869802474975586,
+        "2": 0.5311044454574585
     },
     "sample": {
         "messages": [
@@ -3081,8 +3081,8 @@
         }
     },
     "predict": {
-        "1": 0.7310492396354675,
-        "2": 0.26893800497055054
+        "1": 0.6512240171432495,
+        "2": 0.3485751152038574
     },
     "sample": {
         "messages": [
@@ -3122,8 +3122,8 @@
         }
     },
     "predict": {
-        "1": 0.06008534878492355,
-        "2": 0.9398930072784424
+        "1": 0.2687711715698242,
+        "2": 0.7305957078933716
     },
     "sample": {
         "messages": [
@@ -3158,13 +3158,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 38
         }
     },
     "predict": {
-        "1": 0.2018088847398758,
-        "2": 0.7981696128845215
+        "1": 0.5308924913406372,
+        "2": 0.4685109555721283
     },
     "sample": {
         "messages": [
@@ -3193,19 +3193,19 @@
         "prompt_len": 85,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 39
         }
     },
     "predict": {
-        "1": 0.34864377975463867,
-        "2": 0.6513522863388062
+        "1": 0.5620588660240173,
+        "2": 0.43773189187049866
     },
     "sample": {
         "messages": [
@@ -3234,7 +3234,7 @@
         "prompt_len": 86,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -3245,8 +3245,8 @@
         }
     },
     "predict": {
-        "1": 0.0011695047141984105,
-        "2": 0.99882572889328
+        "1": 0.22264744341373444,
+        "2": 0.7771158814430237
     },
     "sample": {
         "messages": [
@@ -3286,8 +3286,8 @@
         }
     },
     "predict": {
-        "1": 0.29419824481010437,
-        "2": 0.7057448625564575
+        "1": 0.4071022868156433,
+        "2": 0.5923303365707397
     },
     "sample": {
         "messages": [
@@ -3322,13 +3322,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 40
         }
     },
     "predict": {
-        "1": 0.6224275827407837,
-        "2": 0.37752142548561096
+        "1": 0.4685532748699188,
+        "2": 0.5309404134750366
     },
     "sample": {
         "messages": [
@@ -3357,7 +3357,7 @@
         "prompt_len": 91,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "1"
+        "generated_token": "2"
     }
 }
 {
@@ -3368,8 +3368,8 @@
         }
     },
     "predict": {
-        "1": 0.4999956488609314,
-        "2": 0.4999956488609314
+        "1": 0.5618895888328552,
+        "2": 0.4376000463962555
     },
     "sample": {
         "messages": [
@@ -3404,13 +3404,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 41
         }
     },
     "predict": {
-        "1": 0.06754608452320099,
-        "2": 0.9324448704719543
+        "1": 0.4997299611568451,
+        "2": 0.4997299611568451
     },
     "sample": {
         "messages": [
@@ -3439,19 +3439,19 @@
         "prompt_len": 77,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 42
         }
     },
     "predict": {
-        "1": 0.34864187240600586,
-        "2": 0.6513487696647644
+        "1": 0.4997691512107849,
+        "2": 0.4997691512107849
     },
     "sample": {
         "messages": [
@@ -3480,19 +3480,19 @@
         "prompt_len": 84,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 42
         }
     },
     "predict": {
-        "1": 0.18242338299751282,
-        "2": 0.817564845085144
+        "1": 0.49974381923675537,
+        "2": 0.49974381923675537
     },
     "sample": {
         "messages": [
@@ -3521,7 +3521,7 @@
         "prompt_len": 84,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -3532,8 +3532,8 @@
         }
     },
     "predict": {
-        "1": 0.0009110159589909017,
-        "2": 0.9990503191947937
+        "1": 0.32023319602012634,
+        "2": 0.6779336333274841
     },
     "sample": {
         "messages": [
@@ -3573,8 +3573,8 @@
         }
     },
     "predict": {
-        "1": 0.09534304589033127,
-        "2": 0.9045896530151367
+        "1": 0.43682730197906494,
+        "2": 0.5608973503112793
     },
     "sample": {
         "messages": [
@@ -3614,8 +3614,8 @@
         }
     },
     "predict": {
-        "1": 3.288740344942198e-06,
-        "2": 0.9999940395355225
+        "1": 0.04208233952522278,
+        "2": 0.9577896595001221
     },
     "sample": {
         "messages": [
@@ -3655,8 +3655,8 @@
         }
     },
     "predict": {
-        "1": 0.8670345544815063,
-        "2": 0.13296405971050262
+        "1": 0.6223546266555786,
+        "2": 0.37747716903686523
     },
     "sample": {
         "messages": [
@@ -3696,8 +3696,8 @@
         }
     },
     "predict": {
-        "1": 0.04208729416131973,
-        "2": 0.9579023718833923
+        "1": 0.3774266541004181,
+        "2": 0.6222713589668274
     },
     "sample": {
         "messages": [
@@ -3737,8 +3737,8 @@
         }
     },
     "predict": {
-        "1": 0.8933014273643494,
-        "2": 0.10668964684009552
+        "1": 0.6789510846138,
+        "2": 0.3207138180732727
     },
     "sample": {
         "messages": [
@@ -3773,13 +3773,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 46
         }
     },
     "predict": {
-        "1": 0.11920077353715897,
-        "2": 0.8807812333106995
+        "1": 0.5922853350639343,
+        "2": 0.40707138180732727
     },
     "sample": {
         "messages": [
@@ -3808,19 +3808,19 @@
         "prompt_len": 97,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 46
         }
     },
     "predict": {
-        "1": 0.26893293857574463,
-        "2": 0.7310354709625244
+        "1": 0.49966880679130554,
+        "2": 0.49966880679130554
     },
     "sample": {
         "messages": [
@@ -3849,7 +3849,7 @@
         "prompt_len": 97,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -3860,8 +3860,8 @@
         }
     },
     "predict": {
-        "1": 0.4687843322753906,
-        "2": 0.5312022566795349
+        "1": 0.4375786781311035,
+        "2": 0.5618621706962585
     },
     "sample": {
         "messages": [
@@ -3896,13 +3896,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 47
         }
     },
     "predict": {
-        "1": 0.11920198798179626,
-        "2": 0.8807901740074158
+        "1": 0.49976199865341187,
+        "2": 0.49976199865341187
     },
     "sample": {
         "messages": [
@@ -3931,7 +3931,7 @@
         "prompt_len": 84,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -3942,8 +3942,8 @@
         }
     },
     "predict": {
-        "1": 0.0004305548791307956,
-        "2": 0.9995643496513367
+        "1": 0.29415035247802734,
+        "2": 0.7056300044059753
     },
     "sample": {
         "messages": [
@@ -3978,13 +3978,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 48
         }
     },
     "predict": {
-        "1": 0.29421404004096985,
-        "2": 0.7057828307151794
+        "1": 0.5620615482330322,
+        "2": 0.4377339780330658
     },
     "sample": {
         "messages": [
@@ -4013,7 +4013,7 @@
         "prompt_len": 87,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4024,8 +4024,8 @@
         }
     },
     "predict": {
-        "1": 0.1645117849111557,
-        "2": 0.83545982837677
+        "1": 0.468570351600647,
+        "2": 0.5309597849845886
     },
     "sample": {
         "messages": [
@@ -4065,8 +4065,8 @@
         }
     },
     "predict": {
-        "1": 0.06008599326014519,
-        "2": 0.9399030804634094
+        "1": 0.34851348400115967,
+        "2": 0.6511088609695435
     },
     "sample": {
         "messages": [
@@ -4106,8 +4106,8 @@
         }
     },
     "predict": {
-        "1": 0.053403060883283615,
-        "2": 0.946591854095459
+        "1": 0.3773762285709381,
+        "2": 0.6221882104873657
     },
     "sample": {
         "messages": [
@@ -4147,8 +4147,8 @@
         }
     },
     "predict": {
-        "1": 0.00026118988171219826,
-        "2": 0.9997372031211853
+        "1": 0.13292139768600464,
+        "2": 0.8667563796043396
     },
     "sample": {
         "messages": [
@@ -4188,8 +4188,8 @@
         }
     },
     "predict": {
-        "1": 0.01798619143664837,
-        "2": 0.9820127487182617
+        "1": 0.3484882414340973,
+        "2": 0.6510617136955261
     },
     "sample": {
         "messages": [
@@ -4224,13 +4224,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 51
         }
     },
     "predict": {
-        "1": 0.3486437201499939,
-        "2": 0.6513522267341614
+        "1": 0.6221812963485718,
+        "2": 0.37737205624580383
     },
     "sample": {
         "messages": [
@@ -4259,7 +4259,7 @@
         "prompt_len": 79,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4270,8 +4270,8 @@
         }
     },
     "predict": {
-        "1": 0.00017952173948287964,
-        "2": 0.999785840511322
+        "1": 0.3768748641014099,
+        "2": 0.6213616132736206
     },
     "sample": {
         "messages": [
@@ -4311,8 +4311,8 @@
         }
     },
     "predict": {
-        "1": 0.003593507455661893,
-        "2": 0.9963800311088562
+        "1": 0.24472229182720184,
+        "2": 0.7537977695465088
     },
     "sample": {
         "messages": [
@@ -4352,8 +4352,8 @@
         }
     },
     "predict": {
-        "1": 0.04208560660481453,
-        "2": 0.957863986492157
+        "1": 0.3767865002155304,
+        "2": 0.6212159395217896
     },
     "sample": {
         "messages": [
@@ -4393,8 +4393,8 @@
         }
     },
     "predict": {
-        "1": 0.02595537342131138,
-        "2": 0.9739682078361511
+        "1": 0.3766106069087982,
+        "2": 0.6209259033203125
     },
     "sample": {
         "messages": [
@@ -4429,13 +4429,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 54
         }
     },
     "predict": {
-        "1": 0.34864017367362976,
-        "2": 0.6513455510139465
+        "1": 0.5619992613792419,
+        "2": 0.437685489654541
     },
     "sample": {
         "messages": [
@@ -4464,7 +4464,7 @@
         "prompt_len": 93,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4475,8 +4475,8 @@
         }
     },
     "predict": {
-        "1": 0.3208187520503998,
-        "2": 0.6791732311248779
+        "1": 0.46865031123161316,
+        "2": 0.5310503840446472
     },
     "sample": {
         "messages": [
@@ -4516,8 +4516,8 @@
         }
     },
     "predict": {
-        "1": 0.04208741337060928,
-        "2": 0.957905113697052
+        "1": 0.34849637746810913,
+        "2": 0.6510769128799438
     },
     "sample": {
         "messages": [
@@ -4557,8 +4557,8 @@
         }
     },
     "predict": {
-        "1": 0.6224420070648193,
-        "2": 0.37753015756607056
+        "1": 0.561853289604187,
+        "2": 0.43757179379463196
     },
     "sample": {
         "messages": [
@@ -4598,8 +4598,8 @@
         }
     },
     "predict": {
-        "1": 0.00017952758935280144,
-        "2": 0.9998183846473694
+        "1": 0.18239745497703552,
+        "2": 0.817448616027832
     },
     "sample": {
         "messages": [
@@ -4639,8 +4639,8 @@
         }
     },
     "predict": {
-        "1": 0.04208764061331749,
-        "2": 0.9579102993011475
+        "1": 0.3774791359901428,
+        "2": 0.6223578453063965
     },
     "sample": {
         "messages": [
@@ -4680,8 +4680,8 @@
         }
     },
     "predict": {
-        "1": 0.9982919096946716,
-        "2": 0.0017007100395858288
+        "1": 0.6509857773780823,
+        "2": 0.3484475910663605
     },
     "sample": {
         "messages": [
@@ -4721,8 +4721,8 @@
         }
     },
     "predict": {
-        "1": 3.120183464488946e-05,
-        "2": 0.9999662637710571
+        "1": 0.09530383348464966,
+        "2": 0.9042176604270935
     },
     "sample": {
         "messages": [
@@ -4762,8 +4762,8 @@
         }
     },
     "predict": {
-        "1": 0.13296319544315338,
-        "2": 0.8670289516448975
+        "1": 0.4685486853122711,
+        "2": 0.5309352278709412
     },
     "sample": {
         "messages": [
@@ -4798,13 +4798,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 58
         }
     },
     "predict": {
-        "1": 0.34863853454589844,
-        "2": 0.651342511177063
+        "1": 0.49975061416625977,
+        "2": 0.49975061416625977
     },
     "sample": {
         "messages": [
@@ -4833,7 +4833,7 @@
         "prompt_len": 72,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4844,8 +4844,8 @@
         }
     },
     "predict": {
-        "1": 0.022976001724600792,
-        "2": 0.9769644141197205
+        "1": 0.3765953481197357,
+        "2": 0.6209007501602173
     },
     "sample": {
         "messages": [
@@ -4885,8 +4885,8 @@
         }
     },
     "predict": {
-        "1": 0.0015010697534307837,
-        "2": 0.9984240531921387
+        "1": 0.22216132283210754,
+        "2": 0.7754191756248474
     },
     "sample": {
         "messages": [
@@ -4921,13 +4921,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 60
         }
     },
     "predict": {
-        "1": 0.26893940567970276,
-        "2": 0.7310529947280884
+        "1": 0.5310652256011963,
+        "2": 0.46866339445114136
     },
     "sample": {
         "messages": [
@@ -4956,7 +4956,7 @@
         "prompt_len": 94,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4967,8 +4967,8 @@
         }
     },
     "predict": {
-        "1": 0.9046451449394226,
-        "2": 0.09534889459609985
+        "1": 0.6511978507041931,
+        "2": 0.3485611081123352
     },
     "sample": {
         "messages": [
@@ -5003,13 +5003,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 61
         }
     },
     "predict": {
-        "1": 0.3208046853542328,
-        "2": 0.679143488407135
+        "1": 0.5922608971595764,
+        "2": 0.4070545732975006
     },
     "sample": {
         "messages": [
@@ -5038,7 +5038,7 @@
         "prompt_len": 80,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -5049,8 +5049,8 @@
         }
     },
     "predict": {
-        "1": 0.01098682638257742,
-        "2": 0.9890025854110718
+        "1": 0.3484278917312622,
+        "2": 0.6509489417076111
     },
     "sample": {
         "messages": [
@@ -5090,8 +5090,8 @@
         }
     },
     "predict": {
-        "1": 0.3486441969871521,
-        "2": 0.651353120803833
+        "1": 0.4072406589984894,
+        "2": 0.5925316214561462
     },
     "sample": {
         "messages": [
@@ -5131,8 +5131,8 @@
         }
     },
     "predict": {
-        "1": 8.481059921905398e-05,
-        "2": 0.9999099969863892
+        "1": 0.16448116302490234,
+        "2": 0.835304319858551
     },
     "sample": {
         "messages": [
@@ -5167,13 +5167,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 63
         }
     },
     "predict": {
-        "1": 0.29421329498291016,
-        "2": 0.7057809829711914
+        "1": 0.5620059370994568,
+        "2": 0.4376906752586365
     },
     "sample": {
         "messages": [
@@ -5202,19 +5202,19 @@
         "prompt_len": 87,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 63
         }
     },
     "predict": {
-        "1": 0.1480463743209839,
-        "2": 0.8519482016563416
+        "1": 0.5310174226760864,
+        "2": 0.46862125396728516
     },
     "sample": {
         "messages": [
@@ -5243,7 +5243,7 @@
         "prompt_len": 87,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -5254,8 +5254,8 @@
         }
     },
     "predict": {
-        "1": 0.18242448568344116,
-        "2": 0.8175697922706604
+        "1": 0.4071462154388428,
+        "2": 0.592394232749939
     },
     "sample": {
         "messages": [
@@ -5295,8 +5295,8 @@
         }
     },
     "predict": {
-        "1": 1.1478750820970163e-05,
-        "2": 0.999987006187439
+        "1": 0.20173293352127075,
+        "2": 0.7978692054748535
     },
     "sample": {
         "messages": [
@@ -5336,8 +5336,8 @@
         }
     },
     "predict": {
-        "1": 0.29418981075286865,
-        "2": 0.7057247161865234
+        "1": 0.46734532713890076,
+        "2": 0.5295716524124146
     },
     "sample": {
         "messages": [
@@ -5377,8 +5377,8 @@
         }
     },
     "predict": {
-        "1": 0.06754294782876968,
-        "2": 0.9324015378952026
+        "1": 0.40638577938079834,
+        "2": 0.5912877917289734
     },
     "sample": {
         "messages": [
@@ -5418,8 +5418,8 @@
         }
     },
     "predict": {
-        "1": 0.40732821822166443,
-        "2": 0.592659056186676
+        "1": 0.46866509318351746,
+        "2": 0.5310671329498291
     },
     "sample": {
         "messages": [
@@ -5459,8 +5459,8 @@
         }
     },
     "predict": {
-        "1": 0.06754574179649353,
-        "2": 0.9324401021003723
+        "1": 0.3774324953556061,
+        "2": 0.622281014919281
     },
     "sample": {
         "messages": [
@@ -5500,8 +5500,8 @@
         }
     },
     "predict": {
-        "1": 0.651343047618866,
-        "2": 0.3486388325691223
+        "1": 0.5924903750419617,
+        "2": 0.4072122871875763
     },
     "sample": {
         "messages": [
@@ -5541,8 +5541,8 @@
         }
     },
     "predict": {
-        "1": 0.9668970108032227,
-        "2": 0.03308539465069771
+        "1": 0.7308381795883179,
+        "2": 0.2688603699207306
     },
     "sample": {
         "messages": [
@@ -5577,13 +5577,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 68
         }
     },
     "predict": {
-        "1": 0.14804446697235107,
-        "2": 0.8519371747970581
+        "1": 0.4996921420097351,
+        "2": 0.4996921420097351
     },
     "sample": {
         "messages": [
@@ -5612,7 +5612,7 @@
         "prompt_len": 87,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -5623,8 +5623,8 @@
         }
     },
     "predict": {
-        "1": 0.007577181328088045,
-        "2": 0.9924150109291077
+        "1": 0.37733572721481323,
+        "2": 0.6221213936805725
     },
     "sample": {
         "messages": [
@@ -5664,8 +5664,8 @@
         }
     },
     "predict": {
-        "1": 0.8519507050514221,
-        "2": 0.1480468213558197
+        "1": 0.6790658235549927,
+        "2": 0.32076799869537354
     },
     "sample": {
         "messages": [
@@ -5705,8 +5705,8 @@
         }
     },
     "predict": {
-        "1": 0.007577224634587765,
-        "2": 0.9924206137657166
+        "1": 0.40725547075271606,
+        "2": 0.5925531983375549
     },
     "sample": {
         "messages": [
@@ -5746,8 +5746,8 @@
         }
     },
     "predict": {
-        "1": 0.07585762441158295,
-        "2": 0.9241350889205933
+        "1": 0.43762415647506714,
+        "2": 0.5619205236434937
     },
     "sample": {
         "messages": [
@@ -5787,8 +5787,8 @@
         }
     },
     "predict": {
-        "1": 0.26893875002861023,
-        "2": 0.7310512661933899
+        "1": 0.4685721695423126,
+        "2": 0.530961811542511
     },
     "sample": {
         "messages": [
@@ -5828,8 +5828,8 @@
         }
     },
     "predict": {
-        "1": 0.004070112016052008,
-        "2": 0.9959235191345215
+        "1": 0.2449629306793213,
+        "2": 0.7545389533042908
     },
     "sample": {
         "messages": [
@@ -5869,8 +5869,8 @@
         }
     },
     "predict": {
-        "1": 0.001325014280155301,
-        "2": 0.998668909072876
+        "1": 0.14798715710639954,
+        "2": 0.8516073226928711
     },
     "sample": {
         "messages": [
@@ -5910,8 +5910,8 @@
         }
     },
     "predict": {
-        "1": 0.6791690587997437,
-        "2": 0.3208167552947998
+        "1": 0.5309675335884094,
+        "2": 0.46857720613479614
     },
     "sample": {
         "messages": [
@@ -5951,8 +5951,8 @@
         }
     },
     "predict": {
-        "1": 0.009708426892757416,
-        "2": 0.9902864098548889
+        "1": 0.32067590951919556,
+        "2": 0.6788709163665771
     },
     "sample": {
         "messages": [
@@ -5992,8 +5992,8 @@
         }
     },
     "predict": {
-        "1": 0.0003353240608703345,
-        "2": 0.9995869994163513
+        "1": 0.24464736878871918,
+        "2": 0.7535669803619385
     },
     "sample": {
         "messages": [
@@ -6028,13 +6028,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 73
         }
     },
     "predict": {
-        "1": 0.07585203647613525,
-        "2": 0.924066960811615
+        "1": 0.4990968406200409,
+        "2": 0.4990968406200409
     },
     "sample": {
         "messages": [
@@ -6063,7 +6063,7 @@
         "prompt_len": 70,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -6074,8 +6074,8 @@
         }
     },
     "predict": {
-        "1": 0.010986875742673874,
-        "2": 0.9890069961547852
+        "1": 0.29412874579429626,
+        "2": 0.7055782079696655
     },
     "sample": {
         "messages": [
@@ -6115,8 +6115,8 @@
         }
     },
     "predict": {
-        "1": 0.9579020738601685,
-        "2": 0.04208727926015854
+        "1": 0.6511863470077515,
+        "2": 0.34855493903160095
     },
     "sample": {
         "messages": [
@@ -6151,13 +6151,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 75
         }
     },
     "predict": {
-        "1": 0.16451428830623627,
-        "2": 0.8354725241661072
+        "1": 0.49970299005508423,
+        "2": 0.49970299005508423
     },
     "sample": {
         "messages": [
@@ -6186,7 +6186,7 @@
         "prompt_len": 90,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -6197,8 +6197,8 @@
         }
     },
     "predict": {
-        "1": 0.3486413061618805,
-        "2": 0.6513476967811584
+        "1": 0.4685269892215729,
+        "2": 0.5309106707572937
     },
     "sample": {
         "messages": [
@@ -6238,8 +6238,8 @@
         }
     },
     "predict": {
-        "1": 2.430020867905114e-05,
-        "2": 0.9999743700027466
+        "1": 0.08508796244859695,
+        "2": 0.9147818088531494
     },
     "sample": {
         "messages": [
@@ -6279,8 +6279,8 @@
         }
     },
     "predict": {
-        "1": 0.5621755123138428,
-        "2": 0.43782275915145874
+        "1": 0.6223557591438293,
+        "2": 0.37747785449028015
     },
     "sample": {
         "messages": [
@@ -6320,8 +6320,8 @@
         }
     },
     "predict": {
-        "1": 0.2018096148967743,
-        "2": 0.7981725335121155
+        "1": 0.37738266587257385,
+        "2": 0.6221988201141357
     },
     "sample": {
         "messages": [
@@ -6361,8 +6361,8 @@
         }
     },
     "predict": {
-        "1": 0.020332183688879013,
-        "2": 0.9796594977378845
+        "1": 0.29406994581222534,
+        "2": 0.705437183380127
     },
     "sample": {
         "messages": [
@@ -6402,8 +6402,8 @@
         }
     },
     "predict": {
-        "1": 0.0009110494866035879,
-        "2": 0.9990870952606201
+        "1": 0.18236002326011658,
+        "2": 0.8172808885574341
     },
     "sample": {
         "messages": [
@@ -6443,8 +6443,8 @@
         }
     },
     "predict": {
-        "1": 0.04742567986249924,
-        "2": 0.9525702595710754
+        "1": 0.46857959032058716,
+        "2": 0.5309702157974243
     },
     "sample": {
         "messages": [
@@ -6479,13 +6479,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 79
         }
     },
     "predict": {
-        "1": 0.16451598703861237,
-        "2": 0.8354811668395996
+        "1": 0.49989816546440125,
+        "2": 0.49989816546440125
     },
     "sample": {
         "messages": [
@@ -6514,7 +6514,7 @@
         "prompt_len": 95,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -6525,8 +6525,8 @@
         }
     },
     "predict": {
-        "1": 2.7535574190551415e-05,
-        "2": 0.9999681711196899
+        "1": 0.10667160898447037,
+        "2": 0.8931503891944885
     },
     "sample": {
         "messages": [
@@ -6566,8 +6566,8 @@
         }
     },
     "predict": {
-        "1": 0.34864094853401184,
-        "2": 0.6513469815254211
+        "1": 0.40714889764785767,
+        "2": 0.5923981070518494
     },
     "sample": {
         "messages": [
@@ -6607,8 +6607,8 @@
         }
     },
     "predict": {
-        "1": 0.0006263329414650798,
-        "2": 0.9993718266487122
+        "1": 0.20172545313835144,
+        "2": 0.7978396415710449
     },
     "sample": {
         "messages": [
@@ -6648,8 +6648,8 @@
         }
     },
     "predict": {
-        "1": 0.00460954662412405,
-        "2": 0.9953848719596863
+        "1": 0.2688082754611969,
+        "2": 0.7306965589523315
     },
     "sample": {
         "messages": [
@@ -6689,8 +6689,8 @@
         }
     },
     "predict": {
-        "1": 0.2450818568468094,
-        "2": 0.7549052834510803
+        "1": 0.4376171827316284,
+        "2": 0.5619115829467773
     },
     "sample": {
         "messages": [
@@ -6725,13 +6725,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 82
         }
     },
     "predict": {
-        "1": 0.4686311185359955,
-        "2": 0.5310286283493042
+        "1": 0.5290731191635132,
+        "2": 0.4669053554534912
     },
     "sample": {
         "messages": [
@@ -6760,7 +6760,7 @@
         "prompt_len": 85,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -6771,8 +6771,8 @@
         }
     },
     "predict": {
-        "1": 0.02297515980899334,
-        "2": 0.9769286513328552
+        "1": 0.29339078068733215,
+        "2": 0.7038079500198364
     },
     "sample": {
         "messages": [
@@ -6812,8 +6812,8 @@
         }
     },
     "predict": {
-        "1": 0.24508321285247803,
-        "2": 0.7549094557762146
+        "1": 0.40722301602363586,
+        "2": 0.5925059914588928
     },
     "sample": {
         "messages": [
@@ -6853,8 +6853,8 @@
         }
     },
     "predict": {
-        "1": 0.22269901633262634,
-        "2": 0.7772959470748901
+        "1": 0.4686603844165802,
+        "2": 0.5310617685317993
     },
     "sample": {
         "messages": [
@@ -6894,8 +6894,8 @@
         }
     },
     "predict": {
-        "1": 0.010986320674419403,
-        "2": 0.9889569878578186
+        "1": 0.376331090927124,
+        "2": 0.620465099811554
     },
     "sample": {
         "messages": [
@@ -6935,8 +6935,8 @@
         }
     },
     "predict": {
-        "1": 0.16449899971485138,
-        "2": 0.8353949189186096
+        "1": 0.4365803003311157,
+        "2": 0.5605801939964294
     },
     "sample": {
         "messages": [
@@ -6976,8 +6976,8 @@
         }
     },
     "predict": {
-        "1": 0.04208748787641525,
-        "2": 0.9579067826271057
+        "1": 0.4375999867916107,
+        "2": 0.5618895292282104
     },
     "sample": {
         "messages": [
@@ -7012,13 +7012,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 85
         }
     },
     "predict": {
-        "1": 0.29421183466911316,
-        "2": 0.7057775259017944
+        "1": 0.5309422612190247,
+        "2": 0.46855491399765015
     },
     "sample": {
         "messages": [
@@ -7047,7 +7047,7 @@
         "prompt_len": 77,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -7058,8 +7058,8 @@
         }
     },
     "predict": {
-        "1": 0.0046095517463982105,
-        "2": 0.9953859448432922
+        "1": 0.40724000334739685,
+        "2": 0.5925306677818298
     },
     "sample": {
         "messages": [
@@ -7099,8 +7099,8 @@
         }
     },
     "predict": {
-        "1": 0.10668997466564178,
-        "2": 0.8933042287826538
+        "1": 0.46869274973869324,
+        "2": 0.531098484992981
     },
     "sample": {
         "messages": [
@@ -7140,8 +7140,8 @@
         }
     },
     "predict": {
-        "1": 0.004070121794939041,
-        "2": 0.995926022529602
+        "1": 0.2687942385673523,
+        "2": 0.7306584119796753
     },
     "sample": {
         "messages": [
@@ -7176,13 +7176,13 @@
 {
     "metric": {
         "acc": {
-            "val": true,
+            "val": false,
             "id": 87
         }
     },
     "predict": {
-        "1": 0.40732234716415405,
-        "2": 0.5926504731178284
+        "1": 0.4997122287750244,
+        "2": 0.4997122287750244
     },
     "sample": {
         "messages": [
@@ -7211,7 +7211,7 @@
         "prompt_len": 84,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -7222,8 +7222,8 @@
         }
     },
     "predict": {
-        "1": 0.5621723532676697,
-        "2": 0.4378202557563782
+        "1": 0.5310584902763367,
+        "2": 0.4686574637889862
     },
     "sample": {
         "messages": [
@@ -7263,8 +7263,8 @@
         }
     },
     "predict": {
-        "1": 0.0017007191199809313,
-        "2": 0.9982972741127014
+        "1": 0.22263851761817932,
+        "2": 0.7770847678184509
     },
     "sample": {
         "messages": [
@@ -7304,8 +7304,8 @@
         }
     },
     "predict": {
-        "1": 0.7310516238212585,
-        "2": 0.2689388692378998
+        "1": 0.7307744026184082,
+        "2": 0.2688368856906891
     },
     "sample": {
         "messages": [
@@ -7345,8 +7345,8 @@
         }
     },
     "predict": {
-        "1": 3.2887437555473298e-06,
-        "2": 0.9999951124191284
+        "1": 0.06006486713886261,
+        "2": 0.9395726323127747
     },
     "sample": {
         "messages": [
@@ -7386,8 +7386,8 @@
         }
     },
     "predict": {
-        "1": 0.09534507989883423,
-        "2": 0.904608964920044
+        "1": 0.3480902314186096,
+        "2": 0.6503181457519531
     },
     "sample": {
         "messages": [
@@ -7427,8 +7427,8 @@
         }
     },
     "predict": {
-        "1": 0.02931118570268154,
-        "2": 0.9706531763076782
+        "1": 0.37690505385398865,
+        "2": 0.6214113831520081
     },
     "sample": {
         "messages": [
@@ -7468,8 +7468,8 @@
         }
     },
     "predict": {
-        "1": 0.562170147895813,
-        "2": 0.4378185570240021
+        "1": 0.6222553253173828,
+        "2": 0.3774169385433197
     },
     "sample": {
         "messages": [
@@ -7504,13 +7504,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 91
         }
     },
     "predict": {
-        "1": 0.4378180503845215,
-        "2": 0.5621694922447205
+        "1": 0.562000036239624,
+        "2": 0.4376860558986664
     },
     "sample": {
         "messages": [
@@ -7539,7 +7539,7 @@
         "prompt_len": 100,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -7550,8 +7550,8 @@
         }
     },
     "predict": {
-        "1": 0.005911040119826794,
-        "2": 0.9940841794013977
+        "1": 0.29406145215034485,
+        "2": 0.705416738986969
     },
     "sample": {
         "messages": [
@@ -7586,13 +7586,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 92
         }
     },
     "predict": {
-        "1": 0.18242233991622925,
-        "2": 0.8175601959228516
+        "1": 0.5923555493354797,
+        "2": 0.40711963176727295
     },
     "sample": {
         "messages": [
@@ -7621,7 +7621,7 @@
         "prompt_len": 90,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -7632,8 +7632,8 @@
         }
     },
     "predict": {
-        "1": 0.9046250581741333,
-        "2": 0.09534677863121033
+        "1": 0.5922176837921143,
+        "2": 0.4070248603820801
     },
     "sample": {
         "messages": [
@@ -7673,8 +7673,8 @@
         }
     },
     "predict": {
-        "1": 0.0008040774846449494,
-        "2": 0.9991853833198547
+        "1": 0.2225605547428131,
+        "2": 0.7768126726150513
     },
     "sample": {
         "messages": [
@@ -7714,8 +7714,8 @@
         }
     },
     "predict": {
-        "1": 0.9859333634376526,
-        "2": 0.014063583686947823
+        "1": 0.6512224078178406,
+        "2": 0.3485742509365082
     },
     "sample": {
         "messages": [
@@ -7755,8 +7755,8 @@
         }
     },
     "predict": {
-        "1": 0.007577226497232914,
-        "2": 0.9924208521842957
+        "1": 0.3485764265060425,
+        "2": 0.6512265205383301
     },
     "sample": {
         "messages": [
@@ -7796,8 +7796,8 @@
         }
     },
     "predict": {
-        "1": 0.020332276821136475,
-        "2": 0.9796639680862427
+        "1": 0.4072412848472595,
+        "2": 0.5925325751304626
     },
     "sample": {
         "messages": [
@@ -7832,13 +7832,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 95
         }
     },
     "predict": {
-        "1": 0.34864354133605957,
-        "2": 0.6513518691062927
+        "1": 0.5925514698028564,
+        "2": 0.40725430846214294
     },
     "sample": {
         "messages": [
@@ -7867,7 +7867,7 @@
         "prompt_len": 85,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -7878,8 +7878,8 @@
         }
     },
     "predict": {
-        "1": 0.9740384221076965,
-        "2": 0.025957245379686356
+        "1": 0.561974287033081,
+        "2": 0.43766599893569946
     },
     "sample": {
         "messages": [
@@ -7919,8 +7919,8 @@
         }
     },
     "predict": {
-        "1": 0.0019267325988039374,
-        "2": 0.9980721473693848
+        "1": 0.22262296080589294,
+        "2": 0.7770304679870605
     },
     "sample": {
         "messages": [
@@ -7960,8 +7960,8 @@
         }
     },
     "predict": {
-        "1": 0.13296294212341309,
-        "2": 0.8670272827148438
+        "1": 0.294090211391449,
+        "2": 0.7054857611656189
     },
     "sample": {
         "messages": [
@@ -7996,13 +7996,13 @@
 {
     "metric": {
         "acc": {
-            "val": false,
+            "val": true,
             "id": 97
         }
     },
     "predict": {
-        "1": 0.979662299156189,
-        "2": 0.02033224143087864
+        "1": 0.407183438539505,
+        "2": 0.5924484133720398
     },
     "sample": {
         "messages": [
@@ -8031,7 +8031,7 @@
         "prompt_len": 86,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "1"
+        "generated_token": "2"
     }
 }
 {
@@ -8042,8 +8042,8 @@
         }
     },
     "predict": {
-        "1": 0.26893991231918335,
-        "2": 0.731054425239563
+        "1": 0.4685804843902588,
+        "2": 0.5309712886810303
     },
     "sample": {
         "messages": [
@@ -8083,8 +8083,8 @@
         }
     },
     "predict": {
-        "1": 0.13296303153038025,
-        "2": 0.8670278191566467
+        "1": 0.4070993661880493,
+        "2": 0.5923260450363159
     },
     "sample": {
         "messages": [
@@ -8124,8 +8124,8 @@
         }
     },
     "predict": {
-        "1": 0.010986246168613434,
-        "2": 0.9889503121376038
+        "1": 0.32016804814338684,
+        "2": 0.6777957081794739
     },
     "sample": {
         "messages": [
@@ -8165,8 +8165,8 @@
         }
     },
     "predict": {
-        "1": 0.11918793618679047,
-        "2": 0.880686342716217
+        "1": 0.4063665270805359,
+        "2": 0.591259777545929
     },
     "sample": {
         "messages": [

llmtf_eval/darumeru_PARus_params.jsonl

@@ -1,7 +1,7 @@
 {
     "custom_generation_config": null,
     "model_params": {
-        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
         "generation_config": {
             "bos_token_id": 145109,
             "do_sample": true,
@@ -36,7 +36,7 @@
         },
         "load_in_8bit": false,
         "torch_dtype": "auto",
-        "use_flash_attention_2": true,
+        "attn_implementation": "flash_attention_2",
         "device_map": "cuda:0",
         "use_fast_tokenizer": true,
         "leading_space": false,
@@ -47,7 +47,7 @@
     "task_params": {
         "max_len": 4000,
         "few_shot_count": 0,
-        "batch_size": 2,
+        "batch_size": 16,
         "max_sample_per_dataset": 10000000000000,
         "method": "calculate_tokens_proba"
     }

llmtf_eval/darumeru_PARus_total.jsonl

@@ -1,7 +1,7 @@
 {
     "task_name": "darumeru/PARus",
     "results": {
-        "acc": 0.24
+        "acc": 0.44
     },
-    "leaderboard_result": 0.24
+    "leaderboard_result": 0.44
 }

llmtf_eval/darumeru_RCB.jsonl

@@ -7,9 +7,9 @@
         ]
     },
     "predict": {
-        "1": 0.9949106574058533,
-        "2": 0.0019206294091418386,
-        "3": 0.0031665826682001352
+        "1": 0.8427440524101257,
+        "2": 0.07838740944862366,
+        "3": 0.07838740944862366
     },
     "sample": {
         "messages": [
@@ -51,9 +51,9 @@
         ]
     },
     "predict": {
-        "1": 2.4222761567216367e-05,
-        "2": 0.996787428855896,
-        "3": 0.0031725559383630753
+        "1": 0.24942168593406677,
+        "2": 0.5280256867408752,
+        "3": 0.2201138585805893
     },
     "sample": {
         "messages": [
@@ -95,9 +95,9 @@
         ]
     },
     "predict": {
-        "1": 0.8311020731925964,
-        "2": 0.12745362520217896,
-        "3": 0.04137813299894333
+        "1": 0.7282425165176392,
+        "2": 0.184128537774086,
+        "3": 0.08697616308927536
     },
     "sample": {
         "messages": [
@@ -139,9 +139,9 @@
         ]
     },
     "predict": {
-        "1": 0.6286662220954895,
-        "2": 0.262067049741745,
-        "3": 0.10924579203128815
+        "1": 0.5425964593887329,
+        "2": 0.25630444288253784,
+        "3": 0.19961009919643402
     },
     "sample": {
         "messages": [
@@ -183,9 +183,9 @@
         ]
     },
     "predict": {
-        "1": 0.980975866317749,
-        "2": 0.012348663993179798,
-        "3": 0.0066097634844481945
+        "1": 0.904815137386322,
+        "2": 0.05104617774486542,
+        "3": 0.039754804223775864
     },
     "sample": {
         "messages": [
@@ -227,9 +227,9 @@
         ]
     },
     "predict": {
-        "1": 0.03251832723617554,
-        "2": 0.8386587500572205,
-        "3": 0.1286124885082245
+        "1": 0.26357001066207886,
+        "2": 0.43455347418785095,
+        "3": 0.2986639440059662
     },
     "sample": {
         "messages": [
@@ -271,9 +271,9 @@
         ]
     },
     "predict": {
-        "1": 0.359849750995636,
-        "2": 0.28025126457214355,
-        "3": 0.359849750995636
+        "1": 0.51088547706604,
+        "2": 0.27345728874206543,
+        "3": 0.21296873688697815
     },
     "sample": {
         "messages": [
@@ -315,9 +315,9 @@
         ]
     },
     "predict": {
-        "1": 0.02847575768828392,
-        "2": 0.9429875612258911,
-        "3": 0.02847575768828392
+        "1": 0.3646199405193329,
+        "2": 0.4131685495376587,
+        "3": 0.22115319967269897
     },
     "sample": {
         "messages": [
@@ -359,9 +359,9 @@
         ]
     },
     "predict": {
-        "1": 0.7960770130157471,
-        "2": 0.10773730278015137,
-        "3": 0.09507784247398376
+        "1": 0.7268967032432556,
+        "2": 0.12631569802761078,
+        "3": 0.14313443005084991
     },
     "sample": {
         "messages": [
@@ -403,9 +403,9 @@
         ]
     },
     "predict": {
-        "1": 0.6739814281463623,
-        "2": 0.007487257942557335,
-        "3": 0.31836628913879395
+        "1": 0.5728665590286255,
+        "2": 0.07752905786037445,
+        "3": 0.34746116399765015
     },
     "sample": {
         "messages": [
@@ -447,9 +447,9 @@
         ]
     },
     "predict": {
-        "1": 0.9806196689605713,
-        "2": 0.010893701575696468,
-        "3": 0.008484022691845894
+        "1": 0.8923271894454956,
+        "2": 0.0503416582942009,
+        "3": 0.05704456567764282
     },
     "sample": {
         "messages": [
@@ -491,9 +491,9 @@
         ]
     },
     "predict": {
-        "1": 1.88568501471309e-05,
-        "2": 0.9963723421096802,
-        "3": 0.003593479748815298
+        "1": 0.10636842995882034,
+        "2": 0.6936092376708984,
+        "3": 0.19872237741947174
     },
     "sample": {
         "messages": [
@@ -535,9 +535,9 @@
         ]
     },
     "predict": {
-        "1": 0.9927825927734375,
-        "2": 0.0040572755970060825,
-        "3": 0.003159809624776244
+        "1": 0.8976678252220154,
+        "2": 0.04469224810600281,
+        "3": 0.05738598108291626
     },
     "sample": {
         "messages": [
@@ -579,9 +579,9 @@
         ]
     },
     "predict": {
-        "1": 0.9989213943481445,
-        "2": 4.535096013569273e-05,
-        "3": 0.0010321830632165074
+        "1": 0.9036626219749451,
+        "2": 0.04499071091413498,
+        "3": 0.05098116025328636
     },
     "sample": {
         "messages": [
@@ -623,9 +623,9 @@
         ]
     },
     "predict": {
-        "1": 0.8579575419425964,
-        "2": 0.06215023994445801,
-        "3": 0.0798024833202362
+        "1": 0.7032796144485474,
+        "2": 0.17781692743301392,
+        "3": 0.10785142332315445
     },
     "sample": {
         "messages": [
@@ -660,16 +660,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.14122389256954193,
-        "2": 0.717194139957428,
-        "3": 0.14122389256954193
+        "1": 0.40004345774650574,
+        "2": 0.35303711891174316,
+        "3": 0.24263861775398254
     },
     "sample": {
         "messages": [
@@ -699,7 +699,7 @@
         "prompt_len": 101,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -711,9 +711,9 @@
         ]
     },
     "predict": {
-        "1": 0.6938971281051636,
-        "2": 0.05026574432849884,
-        "3": 0.25527051091194153
+        "1": 0.600358784198761,
+        "2": 0.17200568318367004,
+        "3": 0.22085967659950256
     },
     "sample": {
         "messages": [
@@ -755,9 +755,9 @@
         ]
     },
     "predict": {
-        "1": 0.928261399269104,
-        "2": 0.019265450537204742,
-        "3": 0.052368927747011185
+        "1": 0.784460723400116,
+        "2": 0.12030095607042313,
+        "3": 0.09369047731161118
     },
     "sample": {
         "messages": [
@@ -795,13 +795,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.103329136967659,
-        "2": 0.7635048031806946,
-        "3": 0.13267722725868225
+        "1": 0.5102749466896057,
+        "2": 0.27313050627708435,
+        "3": 0.21271422505378723
     },
     "sample": {
         "messages": [
@@ -831,7 +831,7 @@
         "prompt_len": 106,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -843,9 +843,9 @@
         ]
     },
     "predict": {
-        "1": 0.9307658076286316,
-        "2": 0.0017968007596209645,
-        "3": 0.06742445379495621
+        "1": 0.7484683394432068,
+        "2": 0.061438024044036865,
+        "3": 0.1892424374818802
     },
     "sample": {
         "messages": [
@@ -887,9 +887,9 @@
         ]
     },
     "predict": {
-        "1": 0.0007993357139639556,
-        "2": 0.993293046951294,
-        "3": 0.005906336009502411
+        "1": 0.28307968378067017,
+        "2": 0.46671947836875916,
+        "3": 0.24981695413589478
     },
     "sample": {
         "messages": [
@@ -931,9 +931,9 @@
         ]
     },
     "predict": {
-        "1": 0.9381154775619507,
-        "2": 0.015163224190473557,
-        "3": 0.04670601710677147
+        "1": 0.7887781262397766,
+        "2": 0.07336778938770294,
+        "3": 0.13706907629966736
     },
     "sample": {
         "messages": [
@@ -975,9 +975,9 @@
         ]
     },
     "predict": {
-        "1": 0.5897971987724304,
-        "2": 0.13160154223442078,
-        "3": 0.2786004841327667
+        "1": 0.6943702697753906,
+        "2": 0.19894041121006012,
+        "3": 0.10648513585329056
     },
     "sample": {
         "messages": [
@@ -1019,9 +1019,9 @@
         ]
     },
     "predict": {
-        "1": 0.5378125309944153,
-        "2": 0.3261997699737549,
-        "3": 0.1359802931547165
+        "1": 0.48075437545776367,
+        "2": 0.2915922701358795,
+        "3": 0.22709229588508606
     },
     "sample": {
         "messages": [
@@ -1056,16 +1056,16 @@
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.19808343052864075,
-        "2": 0.47517746686935425,
-        "3": 0.3265843689441681
+        "1": 0.4402084946632385,
+        "2": 0.3428347408771515,
+        "3": 0.20793978869915009
     },
     "sample": {
         "messages": [
@@ -1095,7 +1095,7 @@
         "prompt_len": 89,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1103,13 +1103,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.23351947963237762,
-        "2": 0.560184121131897,
-        "3": 0.20608024299144745
+        "1": 0.41778889298439026,
+        "2": 0.3253743350505829,
+        "3": 0.2534017860889435
     },
     "sample": {
         "messages": [
@@ -1139,7 +1139,7 @@
         "prompt_len": 97,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1151,9 +1151,9 @@
         ]
     },
     "predict": {
-        "1": 0.7419307231903076,
-        "2": 0.12892821431159973,
-        "3": 0.12892821431159973
+        "1": 0.5712354183197021,
+        "2": 0.21014578640460968,
+        "3": 0.21014578640460968
     },
     "sample": {
         "messages": [
@@ -1195,9 +1195,9 @@
         ]
     },
     "predict": {
-        "1": 0.7944108247756958,
-        "2": 0.08373028039932251,
-        "3": 0.12182684987783432
+        "1": 0.6559047102928162,
+        "2": 0.21294108033180237,
+        "3": 0.12915529310703278
     },
     "sample": {
         "messages": [
@@ -1239,9 +1239,9 @@
         ]
     },
     "predict": {
-        "1": 0.0012944919290021062,
-        "2": 0.9756640195846558,
-        "3": 0.02294541895389557
+        "1": 0.06279128789901733,
+        "2": 0.7649545073509216,
+        "3": 0.17068442702293396
     },
     "sample": {
         "messages": [
@@ -1283,9 +1283,9 @@
         ]
     },
     "predict": {
-        "1": 0.628616988658905,
-        "2": 0.10923724621534348,
-        "3": 0.26204654574394226
+        "1": 0.4659345746040344,
+        "2": 0.24939681589603424,
+        "3": 0.2826036214828491
     },
     "sample": {
         "messages": [
@@ -1327,9 +1327,9 @@
         ]
     },
     "predict": {
-        "1": 0.9346874952316284,
-        "2": 0.005557854659855366,
-        "3": 0.05975256860256195
+        "1": 0.8445973992347717,
+        "2": 0.05399330332875252,
+        "3": 0.10087277740240097
     },
     "sample": {
         "messages": [
@@ -1364,16 +1364,16 @@
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.19619548320770264,
-        "2": 0.6847895383834839,
-        "3": 0.11899856477975845
+        "1": 0.7727751135826111,
+        "2": 0.09229482710361481,
+        "3": 0.13428816199302673
     },
     "sample": {
         "messages": [
@@ -1403,7 +1403,7 @@
         "prompt_len": 164,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1415,9 +1415,9 @@
         ]
     },
     "predict": {
-        "1": 0.9973662495613098,
-        "2": 0.0007083039381541312,
-        "3": 0.0019253698410466313
+        "1": 0.9634820222854614,
+        "2": 0.013743331655859947,
+        "3": 0.022658925503492355
     },
     "sample": {
         "messages": [
@@ -1459,9 +1459,9 @@
         ]
     },
     "predict": {
-        "1": 0.6337988376617432,
-        "2": 0.06680189818143845,
-        "3": 0.2993853688240051
+        "1": 0.35959210991859436,
+        "2": 0.35959210991859436,
+        "3": 0.2800506353378296
     },
     "sample": {
         "messages": [
@@ -1503,9 +1503,9 @@
         ]
     },
     "predict": {
-        "1": 0.6846823692321777,
-        "2": 0.1961647868156433,
-        "3": 0.11897994577884674
+        "1": 0.5241402983665466,
+        "2": 0.24758636951446533,
+        "3": 0.21849419176578522
     },
     "sample": {
         "messages": [
@@ -1547,9 +1547,9 @@
         ]
     },
     "predict": {
-        "1": 0.8949613571166992,
-        "2": 0.021047474816441536,
-        "3": 0.08324436843395233
+        "1": 0.7656006813049316,
+        "2": 0.08069371432065964,
+        "3": 0.15075570344924927
     },
     "sample": {
         "messages": [
@@ -1587,13 +1587,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.331316202878952,
-        "2": 0.3754304349422455,
-        "3": 0.2923855185508728
+        "1": 0.4284827709197998,
+        "2": 0.3337027430534363,
+        "3": 0.22935031354427338
     },
     "sample": {
         "messages": [
@@ -1623,7 +1623,7 @@
         "prompt_len": 128,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -1635,9 +1635,9 @@
         ]
     },
     "predict": {
-        "1": 0.7969197630882263,
-        "2": 0.10785136371850967,
-        "3": 0.0951784998178482
+        "1": 0.5787113308906555,
+        "2": 0.2733638882637024,
+        "3": 0.1463211327791214
     },
     "sample": {
         "messages": [
@@ -1679,9 +1679,9 @@
         ]
     },
     "predict": {
-        "1": 0.4051330089569092,
-        "2": 0.3155179023742676,
-        "3": 0.2784435749053955
+        "1": 0.5426521301269531,
+        "2": 0.2262110561132431,
+        "3": 0.2262110561132431
     },
     "sample": {
         "messages": [
@@ -1723,9 +1723,9 @@
         ]
     },
     "predict": {
-        "1": 0.26278576254844666,
-        "2": 0.1806098222732544,
-        "3": 0.5563174486160278
+        "1": 0.34257927536964417,
+        "2": 0.26680102944374084,
+        "3": 0.38819319009780884
     },
     "sample": {
         "messages": [
@@ -1760,16 +1760,16 @@
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "3"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.22247056663036346,
-        "2": 0.0010302431182935834,
-        "3": 0.776498556137085
+        "1": 0.8387932181358337,
+        "2": 0.04732147604227066,
+        "3": 0.11351831257343292
     },
     "sample": {
         "messages": [
@@ -1799,7 +1799,7 @@
         "prompt_len": 164,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "3"
+        "generated_token": "1"
     }
 }
 {
@@ -1811,9 +1811,9 @@
         ]
     },
     "predict": {
-        "1": 0.9556446075439453,
-        "2": 0.037054359912872314,
-        "3": 0.007296436000615358
+        "1": 0.8853741884231567,
+        "2": 0.04994939640164375,
+        "3": 0.06413629651069641
     },
     "sample": {
         "messages": [
@@ -1855,9 +1855,9 @@
         ]
     },
     "predict": {
-        "1": 8.937624443206005e-06,
-        "2": 0.9997592568397522,
-        "3": 0.00023050435993354768
+        "1": 0.13088563084602356,
+        "2": 0.7531948089599609,
+        "3": 0.11550617218017578
     },
     "sample": {
         "messages": [
@@ -1899,9 +1899,9 @@
         ]
     },
     "predict": {
-        "1": 0.9998630285263062,
-        "2": 2.7532680178410374e-05,
-        "3": 0.00010889385885093361
+        "1": 0.9762156009674072,
+        "2": 0.00957048125565052,
+        "3": 0.013924967497587204
     },
     "sample": {
         "messages": [
@@ -1943,9 +1943,9 @@
         ]
     },
     "predict": {
-        "1": 0.99820876121521,
-        "2": 0.0017005683621391654,
-        "3": 7.471775461453944e-05
+        "1": 0.9316370487213135,
+        "2": 0.052559368312358856,
+        "3": 0.013289088383316994
     },
     "sample": {
         "messages": [
@@ -1987,9 +1987,9 @@
         ]
     },
     "predict": {
-        "1": 5.134141611051746e-05,
-        "2": 0.9979891777038574,
-        "3": 0.0019265724113211036
+        "1": 0.02406180277466774,
+        "2": 0.796817421913147,
+        "3": 0.17779400944709778
     },
     "sample": {
         "messages": [
@@ -2031,9 +2031,9 @@
         ]
     },
     "predict": {
-        "1": 0.007122146897017956,
-        "2": 0.9328172206878662,
-        "3": 0.059633009135723114
+        "1": 0.1850671023130417,
+        "2": 0.5700468420982361,
+        "3": 0.23763087391853333
     },
     "sample": {
         "messages": [
@@ -2075,9 +2075,9 @@
         ]
     },
     "predict": {
-        "1": 0.8834777474403381,
-        "2": 0.07252027094364166,
-        "3": 0.043985765427351
+        "1": 0.6467720866203308,
+        "2": 0.2379341721534729,
+        "3": 0.1123921275138855
     },
     "sample": {
         "messages": [
@@ -2119,9 +2119,9 @@
         ]
     },
     "predict": {
-        "1": 0.0007971510640345514,
-        "2": 0.9905782341957092,
-        "3": 0.008570181205868721
+        "1": 0.22013789415359497,
+        "2": 0.5280833840370178,
+        "3": 0.24944892525672913
     },
     "sample": {
         "messages": [
@@ -2163,9 +2163,9 @@
         ]
     },
     "predict": {
-        "1": 0.5709625482559204,
-        "2": 0.036500412970781326,
-        "3": 0.39241644740104675
+        "1": 0.3714810311794281,
+        "2": 0.2553149461746216,
+        "3": 0.3714810311794281
     },
     "sample": {
         "messages": [
@@ -2207,9 +2207,9 @@
         ]
     },
     "predict": {
-        "1": 0.9990153312683105,
-        "2": 0.00017938339442480356,
-        "3": 0.0008039406384341419
+        "1": 0.9313455820083618,
+        "2": 0.02190314792096615,
+        "3": 0.04636896401643753
     },
     "sample": {
         "messages": [
@@ -2251,9 +2251,9 @@
         ]
     },
     "predict": {
-        "1": 0.0011647538049146533,
-        "2": 0.9947682619094849,
-        "3": 0.0040653906762599945
+        "1": 0.3544420301914215,
+        "2": 0.45511260628700256,
+        "3": 0.18971915543079376
     },
     "sample": {
         "messages": [
@@ -2295,9 +2295,9 @@
         ]
     },
     "predict": {
-        "1": 0.8055086135864258,
-        "2": 0.014753405936062336,
-        "3": 0.1797332763671875
+        "1": 0.6566400527954102,
+        "2": 0.1293000876903534,
+        "3": 0.213179811835289
     },
     "sample": {
         "messages": [
@@ -2339,9 +2339,9 @@
         ]
     },
     "predict": {
-        "1": 0.959823489189148,
-        "2": 0.007328342646360397,
-        "3": 0.03284335508942604
+        "1": 0.8832191824913025,
+        "2": 0.043972890824079514,
+        "3": 0.07249904423952103
     },
     "sample": {
         "messages": [
@@ -2383,9 +2383,9 @@
         ]
     },
     "predict": {
-        "1": 0.4015943109989166,
-        "2": 0.24357926845550537,
-        "3": 0.3544057309627533
+        "1": 0.5693199634552002,
+        "2": 0.18483112752437592,
+        "3": 0.23732785880565643
     },
     "sample": {
         "messages": [
@@ -2420,16 +2420,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.2965918481349945,
-        "2": 0.6278849244117737,
-        "3": 0.0749901607632637
+        "1": 0.5541829466819763,
+        "2": 0.26177749037742615,
+        "3": 0.1799168586730957
     },
     "sample": {
         "messages": [
@@ -2459,7 +2459,7 @@
         "prompt_len": 111,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -2471,9 +2471,9 @@
         ]
     },
     "predict": {
-        "1": 0.6848835945129395,
-        "2": 0.2223491370677948,
-        "3": 0.0926889106631279
+        "1": 0.5337602496147156,
+        "2": 0.28570127487182617,
+        "3": 0.173286572098732
     },
     "sample": {
         "messages": [
@@ -2515,9 +2515,9 @@
         ]
     },
     "predict": {
-        "1": 0.9814509153366089,
-        "2": 0.013999644666910172,
-        "3": 0.004545019473880529
+        "1": 0.7345989942550659,
+        "2": 0.16391119360923767,
+        "3": 0.09941716492176056
     },
     "sample": {
         "messages": [
@@ -2559,9 +2559,9 @@
         ]
     },
     "predict": {
-        "1": 0.0002600970328785479,
-        "2": 0.9955541491508484,
-        "3": 0.004068602342158556
+        "1": 0.044881440699100494,
+        "2": 0.7955425381660461,
+        "3": 0.1566516011953354
     },
     "sample": {
         "messages": [
@@ -2603,9 +2603,9 @@
         ]
     },
     "predict": {
-        "1": 0.8998140096664429,
-        "2": 0.016480669379234314,
-        "3": 0.08369573950767517
+        "1": 0.7258000373840332,
+        "2": 0.11130504310131073,
+        "3": 0.16194787621498108
     },
     "sample": {
         "messages": [
@@ -2647,9 +2647,9 @@
         ]
     },
     "predict": {
-        "1": 0.9987452030181885,
-        "2": 8.47118062665686e-05,
-        "3": 0.0011694104177877307
+        "1": 0.9448137283325195,
+        "2": 0.013477043248713017,
+        "3": 0.04151221737265587
     },
     "sample": {
         "messages": [
@@ -2691,9 +2691,9 @@
         ]
     },
     "predict": {
-        "1": 0.9984468817710876,
-        "2": 5.1364961109356955e-05,
-        "3": 0.0015011040959507227
+        "1": 0.9257425665855408,
+        "2": 0.021771378815174103,
+        "3": 0.052226826548576355
     },
     "sample": {
         "messages": [
@@ -2735,9 +2735,9 @@
         ]
     },
     "predict": {
-        "1": 0.8783456087112427,
-        "2": 0.0720990002155304,
-        "3": 0.04955286905169487
+        "1": 0.8161900043487549,
+        "2": 0.09747999161481857,
+        "3": 0.08602578938007355
     },
     "sample": {
         "messages": [
@@ -2779,9 +2779,9 @@
         ]
     },
     "predict": {
-        "1": 0.9068573117256165,
-        "2": 0.03516267240047455,
-        "3": 0.05797344818711281
+        "1": 0.5138590335845947,
+        "2": 0.2427298128604889,
+        "3": 0.2427298128604889
     },
     "sample": {
         "messages": [
@@ -2823,9 +2823,9 @@
         ]
     },
     "predict": {
-        "1": 0.8776419758796692,
-        "2": 0.04951317235827446,
-        "3": 0.07204123586416245
+        "1": 0.6807488203048706,
+        "2": 0.17212025821208954,
+        "3": 0.13404738903045654
     },
     "sample": {
         "messages": [
@@ -2867,9 +2867,9 @@
         ]
     },
     "predict": {
-        "1": 0.7253900170326233,
-        "2": 0.11124216020107269,
-        "3": 0.16185639798641205
+        "1": 0.8128647208213806,
+        "2": 0.09708285331726074,
+        "3": 0.08567530661821365
     },
     "sample": {
         "messages": [
@@ -2911,9 +2911,9 @@
         ]
     },
     "predict": {
-        "1": 0.45502758026123047,
-        "2": 0.35437583923339844,
-        "3": 0.1896837055683136
+        "1": 0.4602404236793518,
+        "2": 0.3163183033466339,
+        "3": 0.21740218997001648
     },
     "sample": {
         "messages": [
@@ -2955,9 +2955,9 @@
         ]
     },
     "predict": {
-        "1": 0.9575343728065491,
-        "2": 0.022519050166010857,
-        "3": 0.019872991368174553
+        "1": 0.6141341924667358,
+        "2": 0.22592736780643463,
+        "3": 0.15527743101119995
     },
     "sample": {
         "messages": [
@@ -2999,9 +2999,9 @@
         ]
     },
     "predict": {
-        "1": 0.9337077736854553,
-        "2": 0.013318625278770924,
-        "3": 0.05267618969082832
+        "1": 0.8113856315612793,
+        "2": 0.07547061890363693,
+        "3": 0.10980910807847977
     },
     "sample": {
         "messages": [
@@ -3036,16 +3036,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.2582869827747345,
-        "2": 0.6195982694625854,
-        "3": 0.1220061257481575
+        "1": 0.40465742349624634,
+        "2": 0.3151475191116333,
+        "3": 0.27811670303344727
     },
     "sample": {
         "messages": [
@@ -3075,7 +3075,7 @@
         "prompt_len": 170,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -3083,13 +3083,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "1"
+            "2"
         ]
     },
     "predict": {
-        "1": 0.4955792725086212,
-        "2": 0.4373471736907959,
-        "3": 0.06706935912370682
+        "1": 0.3544919788837433,
+        "2": 0.4551767110824585,
+        "3": 0.18974587321281433
     },
     "sample": {
         "messages": [
@@ -3119,7 +3119,7 @@
         "prompt_len": 152,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "1"
+        "generated_token": "2"
     }
 }
 {
@@ -3131,9 +3131,9 @@
         ]
     },
     "predict": {
-        "1": 0.0001386207150062546,
-        "2": 0.9912692308425903,
-        "3": 0.008576159365475178
+        "1": 0.14761339128017426,
+        "2": 0.6615573167800903,
+        "3": 0.1895393431186676
     },
     "sample": {
         "messages": [
@@ -3175,9 +3175,9 @@
         ]
     },
     "predict": {
-        "1": 0.9124384522438049,
-        "2": 0.06609682738780975,
-        "3": 0.021458497270941734
+        "1": 0.8731182217597961,
+        "2": 0.06324847042560577,
+        "3": 0.06324847042560577
     },
     "sample": {
         "messages": [
@@ -3219,9 +3219,9 @@
         ]
     },
     "predict": {
-        "1": 0.9824451804161072,
-        "2": 0.0016737132100388408,
-        "3": 0.01587974652647972
+        "1": 0.8067973256111145,
+        "2": 0.09635820239782333,
+        "3": 0.09635820239782333
     },
     "sample": {
         "messages": [
@@ -3263,9 +3263,9 @@
         ]
     },
     "predict": {
-        "1": 0.5119481086730957,
-        "2": 0.3987056016921997,
-        "3": 0.08896324038505554
+        "1": 0.6339033842086792,
+        "2": 0.2332000583410263,
+        "3": 0.12482297420501709
     },
     "sample": {
         "messages": [
@@ -3300,16 +3300,16 @@
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.24962082505226135,
-        "2": 0.5988092422485352,
-        "3": 0.15140268206596375
+        "1": 0.6332080364227295,
+        "2": 0.18141713738441467,
+        "3": 0.18141713738441467
     },
     "sample": {
         "messages": [
@@ -3339,21 +3339,21 @@
         "prompt_len": 221,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.11569686979055405,
-        "2": 0.8548907041549683,
-        "3": 0.029252750799059868
+        "1": 0.42262449860572815,
+        "2": 0.3729648292064667,
+        "3": 0.19963368773460388
     },
     "sample": {
         "messages": [
@@ -3383,7 +3383,7 @@
         "prompt_len": 91,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -3395,9 +3395,9 @@
         ]
     },
     "predict": {
-        "1": 0.9447677731513977,
-        "2": 0.047037214040756226,
-        "3": 0.008173842914402485
+        "1": 0.6752980351448059,
+        "2": 0.21923716366291046,
+        "3": 0.10356030613183975
     },
     "sample": {
         "messages": [
@@ -3439,9 +3439,9 @@
         ]
     },
     "predict": {
-        "1": 0.6938379406929016,
-        "2": 0.10640349239110947,
-        "3": 0.1987878978252411
+        "1": 0.6745449304580688,
+        "2": 0.15051132440567017,
+        "3": 0.17055167257785797
     },
     "sample": {
         "messages": [
@@ -3483,9 +3483,9 @@
         ]
     },
     "predict": {
-        "1": 0.7288783192634583,
-        "2": 0.0029787591192871332,
-        "3": 0.2681393623352051
+        "1": 0.6213181614875793,
+        "2": 0.08408626914024353,
+        "3": 0.293489933013916
     },
     "sample": {
         "messages": [
@@ -3527,9 +3527,9 @@
         ]
     },
     "predict": {
-        "1": 3.7148015508137178e-06,
-        "2": 0.9968197345733643,
-        "3": 0.0031726588495075703
+        "1": 0.09423360228538513,
+        "2": 0.6144802570343018,
+        "3": 0.2902599275112152
     },
     "sample": {
         "messages": [
@@ -3571,9 +3571,9 @@
         ]
     },
     "predict": {
-        "1": 0.715835452079773,
-        "2": 0.15972448885440826,
-        "3": 0.1243935376405716
+        "1": 0.5732656121253967,
+        "2": 0.23897266387939453,
+        "3": 0.18611210584640503
     },
     "sample": {
         "messages": [
@@ -3615,9 +3615,9 @@
         ]
     },
     "predict": {
-        "1": 0.9993292093276978,
-        "2": 0.00033523759339004755,
-        "3": 0.00033523759339004755
+        "1": 0.9318123459815979,
+        "2": 0.03188486397266388,
+        "3": 0.036130283027887344
     },
     "sample": {
         "messages": [
@@ -3659,9 +3659,9 @@
         ]
     },
     "predict": {
-        "1": 0.9202016592025757,
-        "2": 0.004261379595845938,
-        "3": 0.07553475350141525
+        "1": 0.616383969783783,
+        "2": 0.1558462679386139,
+        "3": 0.22675500810146332
     },
     "sample": {
         "messages": [
@@ -3703,9 +3703,9 @@
         ]
     },
     "predict": {
-        "1": 0.9761813879013062,
-        "2": 0.017879387363791466,
-        "3": 0.005804586689919233
+        "1": 0.7337620854377747,
+        "2": 0.1855241060256958,
+        "3": 0.07733795046806335
     },
     "sample": {
         "messages": [
@@ -3747,9 +3747,9 @@
         ]
     },
     "predict": {
-        "1": 5.1404014811851084e-05,
-        "2": 0.999206006526947,
-        "3": 0.0007096104673109949
+        "1": 0.04653485119342804,
+        "2": 0.8248499035835266,
+        "3": 0.12649483978748322
     },
     "sample": {
         "messages": [
@@ -3791,9 +3791,9 @@
         ]
     },
     "predict": {
-        "1": 0.5863784551620483,
-        "2": 0.24443891644477844,
-        "3": 0.1680002510547638
+        "1": 0.5722522139549255,
+        "2": 0.21051985025405884,
+        "3": 0.21051985025405884
     },
     "sample": {
         "messages": [
@@ -3835,9 +3835,9 @@
         ]
     },
     "predict": {
-        "1": 0.7358587980270386,
-        "2": 0.05330543592572212,
-        "3": 0.2108270823955536
+        "1": 0.5559288859367371,
+        "2": 0.18048366904258728,
+        "3": 0.26260221004486084
     },
     "sample": {
         "messages": [
@@ -3879,9 +3879,9 @@
         ]
     },
     "predict": {
-        "1": 0.0007938354974612594,
-        "2": 0.9864582419395447,
-        "3": 0.012417676858603954
+        "1": 0.08464287221431732,
+        "2": 0.7087060809135437,
+        "3": 0.20304769277572632
     },
     "sample": {
         "messages": [
@@ -3923,9 +3923,9 @@
         ]
     },
     "predict": {
-        "1": 0.9663904309272766,
-        "2": 0.0005344954552128911,
-        "3": 0.033068060874938965
+        "1": 0.7104495763778687,
+        "2": 0.08485110849142075,
+        "3": 0.20354722440242767
     },
     "sample": {
         "messages": [
@@ -3967,9 +3967,9 @@
         ]
     },
     "predict": {
-        "1": 6.595286686206236e-05,
-        "2": 0.9984310269355774,
-        "3": 0.00150108034722507
+        "1": 0.13575875759124756,
+        "2": 0.6894398927688599,
+        "3": 0.17431770265102386
     },
     "sample": {
         "messages": [
@@ -4007,13 +4007,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 6.58664430375211e-05,
-        "2": 0.9971227049827576,
-        "3": 0.0028007125947624445
+        "1": 0.6457290053367615,
+        "2": 0.14408162236213684,
+        "3": 0.20963750779628754
     },
     "sample": {
         "messages": [
@@ -4043,7 +4043,7 @@
         "prompt_len": 114,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4055,9 +4055,9 @@
         ]
     },
     "predict": {
-        "1": 0.9701799750328064,
-        "2": 0.003964903764426708,
-        "3": 0.02585442177951336
+        "1": 0.9141652584075928,
+        "2": 0.04551360756158829,
+        "3": 0.04016561806201935
     },
     "sample": {
         "messages": [
@@ -4099,9 +4099,9 @@
         ]
     },
     "predict": {
-        "1": 0.9929752349853516,
-        "2": 0.0003331060870550573,
-        "3": 0.006690614391118288
+        "1": 0.7747328877449036,
+        "2": 0.07206138223409653,
+        "3": 0.15255394577980042
     },
     "sample": {
         "messages": [
@@ -4143,9 +4143,9 @@
         ]
     },
     "predict": {
-        "1": 0.9844814538955688,
-        "2": 0.014042872935533524,
-        "3": 0.0013061907375231385
+        "1": 0.8732091784477234,
+        "2": 0.09203556925058365,
+        "3": 0.029879573732614517
     },
     "sample": {
         "messages": [
@@ -4187,9 +4187,9 @@
         ]
     },
     "predict": {
-        "1": 0.000625559245236218,
-        "2": 0.9981372356414795,
-        "3": 0.001168698538094759
+        "1": 0.09942677617073059,
+        "2": 0.7346700429916382,
+        "3": 0.16392704844474792
     },
     "sample": {
         "messages": [
@@ -4231,9 +4231,9 @@
         ]
     },
     "predict": {
-        "1": 0.00037962975329719484,
-        "2": 0.9986867308616638,
-        "3": 0.0009106844081543386
+        "1": 0.15539857745170593,
+        "2": 0.6146132946014404,
+        "3": 0.22610361874103546
     },
     "sample": {
         "messages": [
@@ -4275,9 +4275,9 @@
         ]
     },
     "predict": {
-        "1": 0.9979950189590454,
-        "2": 7.470175478374586e-05,
-        "3": 0.001926583587191999
+        "1": 0.841286301612854,
+        "2": 0.08867092430591583,
+        "3": 0.06905698776245117
     },
     "sample": {
         "messages": [
@@ -4319,9 +4319,9 @@
         ]
     },
     "predict": {
-        "1": 0.9946058392524719,
-        "2": 0.0016944303642958403,
-        "3": 0.0035871088039129972
+        "1": 0.8838312029838562,
+        "2": 0.06402452290058136,
+        "3": 0.049862347543239594
     },
     "sample": {
         "messages": [
@@ -4363,9 +4363,9 @@
         ]
     },
     "predict": {
-        "1": 0.9959226846694946,
-        "2": 5.4001402531866916e-06,
-        "3": 0.004070108290761709
+        "1": 0.8444580435752869,
+        "2": 0.025500424206256866,
+        "3": 0.1295018345117569
     },
     "sample": {
         "messages": [
@@ -4407,9 +4407,9 @@
         ]
     },
     "predict": {
-        "1": 0.9714142680168152,
-        "2": 0.017792074009776115,
-        "3": 0.01079143863171339
+        "1": 0.8658605813980103,
+        "2": 0.07107416540384293,
+        "3": 0.06272273510694504
     },
     "sample": {
         "messages": [
@@ -4451,9 +4451,9 @@
         ]
     },
     "predict": {
-        "1": 0.8966901302337646,
-        "2": 0.008790841326117516,
-        "3": 0.09451044350862503
+        "1": 0.8546397686004639,
+        "2": 0.04255000501871109,
+        "3": 0.10207216441631317
     },
     "sample": {
         "messages": [
@@ -4488,16 +4488,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.0021758012007921934,
-        "2": 0.994655966758728,
-        "3": 0.0031657719518989325
+        "1": 0.4656330645084381,
+        "2": 0.36263540387153625,
+        "3": 0.1712968498468399
     },
     "sample": {
         "messages": [
@@ -4527,7 +4527,7 @@
         "prompt_len": 103,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4539,9 +4539,9 @@
         ]
     },
     "predict": {
-        "1": 0.6097345352172852,
-        "2": 0.2541751563549042,
-        "3": 0.13605016469955444
+        "1": 0.449106901884079,
+        "2": 0.3086663484573364,
+        "3": 0.2403896003961563
     },
     "sample": {
         "messages": [
@@ -4583,9 +4583,9 @@
         ]
     },
     "predict": {
-        "1": 0.6989836692810059,
-        "2": 0.2269267737865448,
-        "3": 0.0736723318696022
+        "1": 0.6791367530822754,
+        "2": 0.1945759356021881,
+        "3": 0.11801625788211823
     },
     "sample": {
         "messages": [
@@ -4620,16 +4620,16 @@
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.1931619942188263,
-        "2": 0.5250687003135681,
-        "3": 0.2810490131378174
+        "1": 0.40359026193618774,
+        "2": 0.31431639194488525,
+        "3": 0.2773832678794861
     },
     "sample": {
         "messages": [
@@ -4659,21 +4659,21 @@
         "prompt_len": 114,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.16358569264411926,
-        "2": 0.8307567834854126,
-        "3": 0.005597595125436783
+        "1": 0.4426115155220032,
+        "2": 0.39060327410697937,
+        "3": 0.16282770037651062
     },
     "sample": {
         "messages": [
@@ -4703,21 +4703,21 @@
         "prompt_len": 101,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.0016701437998563051,
-        "2": 0.9803500175476074,
-        "3": 0.017955737188458443
+        "1": 0.48508769273757935,
+        "2": 0.3333955705165863,
+        "3": 0.17845381796360016
     },
     "sample": {
         "messages": [
@@ -4747,7 +4747,7 @@
         "prompt_len": 145,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4755,13 +4755,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.26198387145996094,
-        "2": 0.6284666657447815,
-        "3": 0.10921111702919006
+        "1": 0.5238513946533203,
+        "2": 0.2803974747657776,
+        "3": 0.19271418452262878
     },
     "sample": {
         "messages": [
@@ -4791,7 +4791,7 @@
         "prompt_len": 101,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -4803,9 +4803,9 @@
         ]
     },
     "predict": {
-        "1": 0.8508579134941101,
-        "2": 0.001279213116504252,
-        "3": 0.1478569209575653
+        "1": 0.5673630833625793,
+        "2": 0.08700795471668243,
+        "3": 0.34412312507629395
     },
     "sample": {
         "messages": [
@@ -4847,9 +4847,9 @@
         ]
     },
     "predict": {
-        "1": 8.93759624887025e-06,
-        "2": 0.9997561573982239,
-        "3": 0.00023050364688970149
+        "1": 0.20961983501911163,
+        "2": 0.6456745862960815,
+        "3": 0.14406947791576385
     },
     "sample": {
         "messages": [
@@ -4891,9 +4891,9 @@
         ]
     },
     "predict": {
-        "1": 0.988469660282135,
-        "2": 0.0005467070732265711,
-        "3": 0.010980906896293163
+        "1": 0.9462574124336243,
+        "2": 0.011911623179912567,
+        "3": 0.04157565161585808
     },
     "sample": {
         "messages": [
@@ -4935,9 +4935,9 @@
         ]
     },
     "predict": {
-        "1": 0.9697875380516052,
-        "2": 0.007404418662190437,
-        "3": 0.02280721813440323
+        "1": 0.879802942276001,
+        "2": 0.06373271346092224,
+        "3": 0.05624391883611679
     },
     "sample": {
         "messages": [
@@ -4979,9 +4979,9 @@
         ]
     },
     "predict": {
-        "1": 0.5741863250732422,
-        "2": 0.23935647308826447,
-        "3": 0.18641100823879242
+        "1": 0.5246575474739075,
+        "2": 0.28082895278930664,
+        "3": 0.19301073253154755
     },
     "sample": {
         "messages": [
@@ -5023,9 +5023,9 @@
         ]
     },
     "predict": {
-        "1": 0.9745687246322632,
-        "2": 0.015752436593174934,
-        "3": 0.009554335847496986
+        "1": 0.8045055270195007,
+        "2": 0.13980208337306976,
+        "3": 0.051430314779281616
     },
     "sample": {
         "messages": [
@@ -5067,9 +5067,9 @@
         ]
     },
     "predict": {
-        "1": 0.006572623271495104,
-        "2": 0.9754637479782104,
-        "3": 0.017866242676973343
+        "1": 0.14742936193943024,
+        "2": 0.6607325077056885,
+        "3": 0.18930304050445557
     },
     "sample": {
         "messages": [
@@ -5104,16 +5104,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.35423046350479126,
-        "2": 0.5154022574424744,
-        "3": 0.13031409680843353
+        "1": 0.37023016810417175,
+        "2": 0.37023016810417175,
+        "3": 0.25445523858070374
     },
     "sample": {
         "messages": [
@@ -5143,7 +5143,7 @@
         "prompt_len": 95,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -5155,9 +5155,9 @@
         ]
     },
     "predict": {
-        "1": 0.9688199758529663,
-        "2": 0.008381934836506844,
-        "3": 0.022784462198615074
+        "1": 0.6989871263504028,
+        "2": 0.17673161625862122,
+        "3": 0.12146573513746262
     },
     "sample": {
         "messages": [
@@ -5199,9 +5199,9 @@
         ]
     },
     "predict": {
-        "1": 0.996656060218811,
-        "2": 0.0008020420791581273,
-        "3": 0.002470463514328003
+        "1": 0.9553428888320923,
+        "2": 0.01749771647155285,
+        "3": 0.02545902505517006
     },
     "sample": {
         "messages": [
@@ -5243,9 +5243,9 @@
         ]
     },
     "predict": {
-        "1": 0.7234998941421509,
-        "2": 0.01032017171382904,
-        "3": 0.2661607563495636
+        "1": 0.6160777807235718,
+        "2": 0.15576885640621185,
+        "3": 0.22664237022399902
     },
     "sample": {
         "messages": [
@@ -5287,9 +5287,9 @@
         ]
     },
     "predict": {
-        "1": 0.9639463424682617,
-        "2": 0.0030680301133543253,
-        "3": 0.032984428107738495
+        "1": 0.8549993634223938,
+        "2": 0.0425679087638855,
+        "3": 0.10211510956287384
     },
     "sample": {
         "messages": [
@@ -5331,9 +5331,9 @@
         ]
     },
     "predict": {
-        "1": 0.4442080557346344,
-        "2": 0.20982903242111206,
-        "3": 0.34594959020614624
+        "1": 0.542923629283905,
+        "2": 0.19973047077655792,
+        "3": 0.25645899772644043
     },
     "sample": {
         "messages": [
@@ -5375,9 +5375,9 @@
         ]
     },
     "predict": {
-        "1": 0.9747047424316406,
-        "2": 0.007441961672157049,
-        "3": 0.017852339893579483
+        "1": 0.9360924363136292,
+        "2": 0.017145130783319473,
+        "3": 0.046605296432971954
     },
     "sample": {
         "messages": [
@@ -5419,9 +5419,9 @@
         ]
     },
     "predict": {
-        "1": 0.3836464583873749,
-        "2": 0.3836464583873749,
-        "3": 0.23269334435462952
+        "1": 0.48258522152900696,
+        "2": 0.2583092749118805,
+        "3": 0.2583092749118805
     },
     "sample": {
         "messages": [
@@ -5463,9 +5463,9 @@
         ]
     },
     "predict": {
-        "1": 0.8900082111358643,
-        "2": 0.10629633069038391,
-        "3": 0.003637260291725397
+        "1": 0.5264328122138977,
+        "2": 0.3618116080760956,
+        "3": 0.10366075485944748
     },
     "sample": {
         "messages": [
@@ -5507,9 +5507,9 @@
         ]
     },
     "predict": {
-        "1": 0.8241075277328491,
-        "2": 0.07665394246578217,
-        "3": 0.09842561185359955
+        "1": 0.5575301647186279,
+        "2": 0.20510391891002655,
+        "3": 0.2324131578207016
     },
     "sample": {
         "messages": [
@@ -5551,9 +5551,9 @@
         ]
     },
     "predict": {
-        "1": 0.02890811860561371,
-        "2": 0.9573054313659668,
-        "3": 0.013655227608978748
+        "1": 0.3523302376270294,
+        "2": 0.39924246072769165,
+        "3": 0.24215279519557953
     },
     "sample": {
         "messages": [
@@ -5595,9 +5595,9 @@
         ]
     },
     "predict": {
-        "1": 0.023534899577498436,
-        "2": 0.7793688774108887,
-        "3": 0.19705531001091003
+        "1": 0.29909470677375793,
+        "2": 0.4351802170276642,
+        "3": 0.2639501392841339
     },
     "sample": {
         "messages": [
@@ -5639,9 +5639,9 @@
         ]
     },
     "predict": {
-        "1": 0.8651604652404785,
-        "2": 0.043073803186416626,
-        "3": 0.0911872386932373
+        "1": 0.8119111061096191,
+        "2": 0.07551949471235275,
+        "3": 0.10988021641969681
     },
     "sample": {
         "messages": [
@@ -5683,9 +5683,9 @@
         ]
     },
     "predict": {
-        "1": 0.8635392785072327,
-        "2": 0.003998980391770601,
-        "3": 0.13242805004119873
+        "1": 0.5787488222122192,
+        "2": 0.14633060991764069,
+        "3": 0.2733815908432007
     },
     "sample": {
         "messages": [
@@ -5727,9 +5727,9 @@
         ]
     },
     "predict": {
-        "1": 0.43489623069763184,
-        "2": 0.18129171431064606,
-        "3": 0.3837945759296417
+        "1": 0.4806901216506958,
+        "2": 0.2270619422197342,
+        "3": 0.2915532886981964
     },
     "sample": {
         "messages": [
@@ -5771,9 +5771,9 @@
         ]
     },
     "predict": {
-        "1": 0.9524779915809631,
-        "2": 0.010581075213849545,
-        "3": 0.03693157806992531
+        "1": 0.8475168347358704,
+        "2": 0.03723729029297829,
+        "3": 0.11469893157482147
     },
     "sample": {
         "messages": [
@@ -5815,9 +5815,9 @@
         ]
     },
     "predict": {
-        "1": 0.9840602874755859,
-        "2": 0.014036864973604679,
-        "3": 0.0018996832659468055
+        "1": 0.8955039978027344,
+        "2": 0.07350744307041168,
+        "3": 0.030642462894320488
     },
     "sample": {
         "messages": [
@@ -5859,9 +5859,9 @@
         ]
     },
     "predict": {
-        "1": 0.8148273229598999,
-        "2": 0.024605652317404747,
-        "3": 0.16044899821281433
+        "1": 0.7252457737922668,
+        "2": 0.11122004687786102,
+        "3": 0.16182421147823334
     },
     "sample": {
         "messages": [
@@ -5896,16 +5896,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.32032930850982666,
-        "2": 0.528133749961853,
-        "3": 0.15131285786628723
+        "1": 0.5111352801322937,
+        "2": 0.31001922488212585,
+        "3": 0.16594134271144867
     },
     "sample": {
         "messages": [
@@ -5935,7 +5935,7 @@
         "prompt_len": 90,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -5943,13 +5943,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.01377924345433712,
-        "2": 0.9659996032714844,
-        "3": 0.020048681646585464
+        "1": 0.4119296967983246,
+        "2": 0.36352667212486267,
+        "3": 0.22049008309841156
     },
     "sample": {
         "messages": [
@@ -5979,7 +5979,7 @@
         "prompt_len": 97,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -5991,9 +5991,9 @@
         ]
     },
     "predict": {
-        "1": 0.9683293104171753,
-        "2": 0.0057578966952860355,
-        "3": 0.025805102661252022
+        "1": 0.8117878437042236,
+        "2": 0.09695423394441605,
+        "3": 0.08556180447340012
     },
     "sample": {
         "messages": [
@@ -6035,9 +6035,9 @@
         ]
     },
     "predict": {
-        "1": 0.8739377856254578,
-        "2": 0.033886246383190155,
-        "3": 0.09211236238479614
+        "1": 0.6461917757987976,
+        "2": 0.11229129135608673,
+        "3": 0.23772069811820984
     },
     "sample": {
         "messages": [
@@ -6079,9 +6079,9 @@
         ]
     },
     "predict": {
-        "1": 0.8764171004295349,
-        "2": 0.029989343136548996,
-        "3": 0.09237368404865265
+        "1": 0.8521672487258911,
+        "2": 0.05447722598910332,
+        "3": 0.08981776237487793
     },
     "sample": {
         "messages": [
@@ -6123,9 +6123,9 @@
         ]
     },
     "predict": {
-        "1": 0.9496769905090332,
-        "2": 0.003022613935172558,
-        "3": 0.04728163033723831
+        "1": 0.6733472943305969,
+        "2": 0.1325899362564087,
+        "3": 0.19291724264621735
     },
     "sample": {
         "messages": [
@@ -6167,9 +6167,9 @@
         ]
     },
     "predict": {
-        "1": 6.1392297538986895e-06,
-        "2": 0.9991890788078308,
-        "3": 0.000804080453235656
+        "1": 0.15595495700836182,
+        "2": 0.6168138384819031,
+        "3": 0.22691313922405243
     },
     "sample": {
         "messages": [
@@ -6207,13 +6207,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.07572191208600998,
-        "2": 0.7184295058250427,
-        "3": 0.20583350956439972
+        "1": 0.5484347343444824,
+        "2": 0.1571291983127594,
+        "3": 0.29355597496032715
     },
     "sample": {
         "messages": [
@@ -6243,21 +6243,21 @@
         "prompt_len": 230,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "3"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.22240525484085083,
-        "2": 0.0013224700232967734,
-        "3": 0.7762705683708191
+        "1": 0.7922748327255249,
+        "2": 0.030719827860593796,
+        "3": 0.17678040266036987
     },
     "sample": {
         "messages": [
@@ -6287,21 +6287,21 @@
         "prompt_len": 126,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "3"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "3"
         ]
     },
     "predict": {
-        "1": 0.003063197946175933,
-        "2": 0.8493398427963257,
-        "3": 0.14759312570095062
+        "1": 0.24063578248023987,
+        "2": 0.3089824616909027,
+        "3": 0.4495667815208435
     },
     "sample": {
         "messages": [
@@ -6331,7 +6331,7 @@
         "prompt_len": 108,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "3"
     }
 }
 {
@@ -6343,9 +6343,9 @@
         ]
     },
     "predict": {
-        "1": 0.8214573264122009,
-        "2": 0.12597456574440002,
-        "3": 0.052514009177684784
+        "1": 0.5717660784721375,
+        "2": 0.21034102141857147,
+        "3": 0.21034102141857147
     },
     "sample": {
         "messages": [
@@ -6387,9 +6387,9 @@
         ]
     },
     "predict": {
-        "1": 0.7958472371101379,
-        "2": 0.0950504019856453,
-        "3": 0.1077062115073204
+        "1": 0.9258212447166443,
+        "2": 0.035897985100746155,
+        "3": 0.035897985100746155
     },
     "sample": {
         "messages": [
@@ -6431,9 +6431,9 @@
         ]
     },
     "predict": {
-        "1": 0.7490409016609192,
-        "2": 0.06148502230644226,
-        "3": 0.18938720226287842
+        "1": 0.36907562613487244,
+        "2": 0.36907562613487244,
+        "3": 0.25366172194480896
     },
     "sample": {
         "messages": [
@@ -6475,9 +6475,9 @@
         ]
     },
     "predict": {
-        "1": 0.7041969299316406,
-        "2": 0.1386645883321762,
-        "3": 0.15712757408618927
+        "1": 0.55531245470047,
+        "2": 0.26231104135513306,
+        "3": 0.1802835613489151
     },
     "sample": {
         "messages": [
@@ -6519,9 +6519,9 @@
         ]
     },
     "predict": {
-        "1": 0.008256481029093266,
-        "2": 0.954319417476654,
-        "3": 0.03700298070907593
+        "1": 0.24863293766975403,
+        "2": 0.46450746059417725,
+        "3": 0.2817380130290985
     },
     "sample": {
         "messages": [
@@ -6563,9 +6563,9 @@
         ]
     },
     "predict": {
-        "1": 0.6154287457466125,
-        "2": 0.01127197127789259,
-        "3": 0.37327641248703003
+        "1": 0.5585140585899353,
+        "2": 0.14121447503566742,
+        "3": 0.29895102977752686
     },
     "sample": {
         "messages": [
@@ -6607,9 +6607,9 @@
         ]
     },
     "predict": {
-        "1": 0.9862869381904602,
-        "2": 0.008533054031431675,
-        "3": 0.005175558850169182
+        "1": 0.850590705871582,
+        "2": 0.06982073932886124,
+        "3": 0.07911725342273712
     },
     "sample": {
         "messages": [
@@ -6651,9 +6651,9 @@
         ]
     },
     "predict": {
-        "1": 0.970179557800293,
-        "2": 0.003964902367442846,
-        "3": 0.025854408740997314
+        "1": 0.8748275637626648,
+        "2": 0.04355509951710701,
+        "3": 0.08137163519859314
     },
     "sample": {
         "messages": [
@@ -6695,9 +6695,9 @@
         ]
     },
     "predict": {
-        "1": 0.9508241415023804,
-        "2": 0.001835522474721074,
-        "3": 0.04733874648809433
+        "1": 0.7593733072280884,
+        "2": 0.09069421142339706,
+        "3": 0.14952945709228516
     },
     "sample": {
         "messages": [
@@ -6739,9 +6739,9 @@
         ]
     },
     "predict": {
-        "1": 0.9564642906188965,
-        "2": 0.006444605998694897,
-        "3": 0.03708614408969879
+        "1": 0.6766987442970276,
+        "2": 0.15099190175533295,
+        "3": 0.17109623551368713
     },
     "sample": {
         "messages": [
@@ -6776,16 +6776,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "3",
-            "3"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.3263939917087555,
-        "2": 0.19796794652938843,
-        "3": 0.47490042448043823
+        "1": 0.5178438425064087,
+        "2": 0.27718183398246765,
+        "3": 0.19050411880016327
     },
     "sample": {
         "messages": [
@@ -6815,7 +6815,7 @@
         "prompt_len": 87,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "3"
+        "generated_token": "1"
     }
 }
 {
@@ -6827,9 +6827,9 @@
         ]
     },
     "predict": {
-        "1": 0.0021637813188135624,
-        "2": 0.9891611933708191,
-        "3": 0.008557921275496483
+        "1": 0.3446602523326874,
+        "2": 0.4425525367259979,
+        "3": 0.20904701948165894
     },
     "sample": {
         "messages": [
@@ -6871,9 +6871,9 @@
         ]
     },
     "predict": {
-        "1": 0.3720066547393799,
-        "2": 0.25567618012428284,
-        "3": 0.3720066547393799
+        "1": 0.5550274848937988,
+        "2": 0.23136986792087555,
+        "3": 0.20418322086334229
     },
     "sample": {
         "messages": [
@@ -6915,9 +6915,9 @@
         ]
     },
     "predict": {
-        "1": 0.5375309586524963,
-        "2": 0.28771960735321045,
-        "3": 0.17451074719429016
+        "1": 0.5104585289955139,
+        "2": 0.2732287645339966,
+        "3": 0.21279077231884003
     },
     "sample": {
         "messages": [
@@ -6959,9 +6959,9 @@
         ]
     },
     "predict": {
-        "1": 0.9285780787467957,
-        "2": 0.011689072474837303,
-        "3": 0.059362009167671204
+        "1": 0.8524977564811707,
+        "2": 0.05449835956096649,
+        "3": 0.08985260128974915
     },
     "sample": {
         "messages": [
@@ -7003,9 +7003,9 @@
         ]
     },
     "predict": {
-        "1": 0.6845002174377441,
-        "2": 0.030074842274188995,
-        "3": 0.28534215688705444
+        "1": 0.5477773547172546,
+        "2": 0.15694084763526917,
+        "3": 0.293204128742218
     },
     "sample": {
         "messages": [
@@ -7047,9 +7047,9 @@
         ]
     },
     "predict": {
-        "1": 0.996752917766571,
-        "2": 7.460878987330943e-05,
-        "3": 0.0031724462751299143
+        "1": 0.9256893396377563,
+        "2": 0.014962374232709408,
+        "3": 0.05917733907699585
     },
     "sample": {
         "messages": [
@@ -7091,9 +7091,9 @@
         ]
     },
     "predict": {
-        "1": 0.0034236812498420477,
-        "2": 0.9492919445037842,
-        "3": 0.04726245999336243
+        "1": 0.25393006205558777,
+        "2": 0.41865989565849304,
+        "3": 0.3260526657104492
     },
     "sample": {
         "messages": [
@@ -7131,13 +7131,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.34595102071762085,
-        "2": 0.44420990347862244,
-        "3": 0.2098299115896225
+        "1": 0.6281933784484863,
+        "2": 0.2310994565486908,
+        "3": 0.14016889035701752
     },
     "sample": {
         "messages": [
@@ -7167,7 +7167,7 @@
         "prompt_len": 106,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -7179,9 +7179,9 @@
         ]
     },
     "predict": {
-        "1": 0.9988577365875244,
-        "2": 0.00010878437024075538,
-        "3": 0.0010321172885596752
+        "1": 0.8853259086608887,
+        "2": 0.04994667321443558,
+        "3": 0.06413279473781586
     },
     "sample": {
         "messages": [
@@ -7223,9 +7223,9 @@
         ]
     },
     "predict": {
-        "1": 0.6895111203193665,
-        "2": 0.13577277958393097,
-        "3": 0.17433570325374603
+        "1": 0.5514762997627258,
+        "2": 0.260498970746994,
+        "3": 0.1790381371974945
     },
     "sample": {
         "messages": [
@@ -7267,9 +7267,9 @@
         ]
     },
     "predict": {
-        "1": 0.7882348299026489,
-        "2": 0.03463262319564819,
-        "3": 0.17587897181510925
+        "1": 0.7609981298446655,
+        "2": 0.10298989713191986,
+        "3": 0.1322416365146637
     },
     "sample": {
         "messages": [
@@ -7311,9 +7311,9 @@
         ]
     },
     "predict": {
-        "1": 0.8739826679229736,
-        "2": 0.09211709350347519,
-        "3": 0.03388798609375954
+        "1": 0.5548850893974304,
+        "2": 0.2621091604232788,
+        "3": 0.18014481663703918
     },
     "sample": {
         "messages": [
@@ -7355,9 +7355,9 @@
         ]
     },
     "predict": {
-        "1": 0.005791477393358946,
-        "2": 0.9739767909049988,
-        "3": 0.020214242860674858
+        "1": 0.20582358539104462,
+        "2": 0.4937451183795929,
+        "3": 0.29947155714035034
     },
     "sample": {
         "messages": [
@@ -7399,9 +7399,9 @@
         ]
     },
     "predict": {
-        "1": 0.916553258895874,
-        "2": 0.024425316601991653,
-        "3": 0.05859328806400299
+        "1": 0.6618460416793823,
+        "2": 0.16734088957309723,
+        "3": 0.16734088957309723
     },
     "sample": {
         "messages": [
@@ -7443,9 +7443,9 @@
         ]
     },
     "predict": {
-        "1": 0.6508558392524719,
-        "2": 0.07773364335298538,
-        "3": 0.2713170647621155
+        "1": 0.5680178999900818,
+        "2": 0.1267419308423996,
+        "3": 0.3040381073951721
     },
     "sample": {
         "messages": [
@@ -7487,9 +7487,9 @@
         ]
     },
     "predict": {
-        "1": 0.38364943861961365,
-        "2": 0.38364943861961365,
-        "3": 0.23269516229629517
+        "1": 0.6336524486541748,
+        "2": 0.1602124273777008,
+        "3": 0.2057168334722519
     },
     "sample": {
         "messages": [
@@ -7531,9 +7531,9 @@
         ]
     },
     "predict": {
-        "1": 0.009539642371237278,
-        "2": 0.8587311506271362,
-        "3": 0.13169069588184357
+        "1": 0.2308199256658554,
+        "2": 0.431228369474411,
+        "3": 0.33584100008010864
     },
     "sample": {
         "messages": [
@@ -7575,9 +7575,9 @@
         ]
     },
     "predict": {
-        "1": 0.9666678309440613,
-        "2": 0.01770513877272606,
-        "3": 0.015624729916453362
+        "1": 0.8781532645225525,
+        "2": 0.07208320498466492,
+        "3": 0.04954201728105545
     },
     "sample": {
         "messages": [
@@ -7619,9 +7619,9 @@
         ]
     },
     "predict": {
-        "1": 0.9132862091064453,
-        "2": 0.0017630571965128183,
-        "3": 0.08494885265827179
+        "1": 0.7593562006950378,
+        "2": 0.07063112407922745,
+        "3": 0.1694352775812149
     },
     "sample": {
         "messages": [
@@ -7663,9 +7663,9 @@
         ]
     },
     "predict": {
-        "1": 0.9931216835975647,
-        "2": 0.0059053171426057816,
-        "3": 0.0009056097478605807
+        "1": 0.8923550844192505,
+        "2": 0.0732489675283432,
+        "3": 0.030534710735082626
     },
     "sample": {
         "messages": [
@@ -7700,16 +7700,16 @@
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.3257984519004822,
-        "2": 0.5371508002281189,
-        "3": 0.13581299781799316
+        "1": 0.6177648901939392,
+        "2": 0.20055890083312988,
+        "3": 0.17699261009693146
     },
     "sample": {
         "messages": [
@@ -7739,7 +7739,7 @@
         "prompt_len": 126,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -7751,9 +7751,9 @@
         ]
     },
     "predict": {
-        "1": 0.46300792694091797,
-        "2": 0.21870946884155273,
-        "3": 0.3182203769683838
+        "1": 0.4607653319835663,
+        "2": 0.31667909026145935,
+        "3": 0.2176501452922821
     },
     "sample": {
         "messages": [
@@ -7795,9 +7795,9 @@
         ]
     },
     "predict": {
-        "1": 0.4980303943157196,
-        "2": 0.26657646894454956,
-        "3": 0.23525291681289673
+        "1": 0.40412601828575134,
+        "2": 0.40412601828575134,
+        "3": 0.19089563190937042
     },
     "sample": {
         "messages": [
@@ -7832,16 +7832,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.15115134418010712,
-        "2": 0.7676099538803101,
-        "3": 0.0809054896235466
+        "1": 0.8761624693870544,
+        "2": 0.07191979140043259,
+        "3": 0.04942970722913742
     },
     "sample": {
         "messages": [
@@ -7871,7 +7871,7 @@
         "prompt_len": 114,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -7883,9 +7883,9 @@
         ]
     },
     "predict": {
-        "1": 0.9888356328010559,
-        "2": 0.0001775555283529684,
-        "3": 0.01098497211933136
+        "1": 0.7257519364356995,
+        "2": 0.11129766702651978,
+        "3": 0.1619371473789215
     },
     "sample": {
         "messages": [
@@ -7927,9 +7927,9 @@
         ]
     },
     "predict": {
-        "1": 0.8344322443008423,
-        "2": 0.07761428505182266,
-        "3": 0.0879485085606575
+        "1": 0.6352789998054504,
+        "2": 0.1820105016231537,
+        "3": 0.1820105016231537
     },
     "sample": {
         "messages": [
@@ -7967,13 +7967,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.010796502232551575,
-        "2": 0.857672393321991,
-        "3": 0.1315283179283142
+        "1": 0.7201007008552551,
+        "2": 0.09745503216981888,
+        "3": 0.18206997215747833
     },
     "sample": {
         "messages": [
@@ -8003,7 +8003,7 @@
         "prompt_len": 116,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -8011,13 +8011,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.1164684072136879,
-        "2": 0.8605915904045105,
-        "3": 0.022933989763259888
+        "1": 0.6284369230270386,
+        "2": 0.26197147369384766,
+        "3": 0.1092059537768364
     },
     "sample": {
         "messages": [
@@ -8047,21 +8047,21 @@
         "prompt_len": 145,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.22721615433692932,
-        "2": 0.4810165762901306,
-        "3": 0.29175132513046265
+        "1": 0.5598638653755188,
+        "2": 0.2059624344110489,
+        "3": 0.23338598012924194
     },
     "sample": {
         "messages": [
@@ -8091,7 +8091,7 @@
         "prompt_len": 129,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -8103,9 +8103,9 @@
         ]
     },
     "predict": {
-        "1": 0.9846836924552917,
-        "2": 0.014045758172869682,
-        "3": 0.001152946031652391
+        "1": 0.8510578274726868,
+        "2": 0.07916070520877838,
+        "3": 0.06165042519569397
     },
     "sample": {
         "messages": [
@@ -8147,9 +8147,9 @@
         ]
     },
     "predict": {
-        "1": 0.700525164604187,
-        "2": 0.17712049186229706,
-        "3": 0.12173300981521606
+        "1": 0.7012267112731934,
+        "2": 0.1564648300409317,
+        "3": 0.1380797177553177
     },
     "sample": {
         "messages": [
@@ -8191,9 +8191,9 @@
         ]
     },
     "predict": {
-        "1": 0.8698589205741882,
-        "2": 0.11772260069847107,
-        "3": 0.012407870963215828
+        "1": 0.5885195136070251,
+        "2": 0.2779969573020935,
+        "3": 0.13131646811962128
     },
     "sample": {
         "messages": [
@@ -8235,9 +8235,9 @@
         ]
     },
     "predict": {
-        "1": 0.994073212146759,
-        "2": 0.004603472538292408,
-        "3": 0.0013189169112592936
+        "1": 0.901430606842041,
+        "2": 0.05762653052806854,
+        "3": 0.03960609808564186
     },
     "sample": {
         "messages": [
@@ -8279,9 +8279,9 @@
         ]
     },
     "predict": {
-        "1": 0.06515555083751678,
-        "2": 0.899444580078125,
-        "3": 0.03487525135278702
+        "1": 0.3171122670173645,
+        "2": 0.46139562129974365,
+        "3": 0.21794787049293518
     },
     "sample": {
         "messages": [
@@ -8323,9 +8323,9 @@
         ]
     },
     "predict": {
-        "1": 0.8127654790878296,
-        "2": 0.04585309326648712,
-        "3": 0.14123745262622833
+        "1": 0.6369884610176086,
+        "2": 0.12543044984340668,
+        "3": 0.23433497548103333
     },
     "sample": {
         "messages": [
@@ -8367,9 +8367,9 @@
         ]
     },
     "predict": {
-        "1": 0.48418858647346497,
-        "2": 0.37708646059036255,
-        "3": 0.13872236013412476
+        "1": 0.6911755204200745,
+        "2": 0.15422211587429047,
+        "3": 0.15422211587429047
     },
     "sample": {
         "messages": [
@@ -8411,9 +8411,9 @@
         ]
     },
     "predict": {
-        "1": 0.9524831771850586,
-        "2": 0.01058113295584917,
-        "3": 0.036931779235601425
+        "1": 0.9095896482467651,
+        "2": 0.024239743128418922,
+        "3": 0.06589046120643616
     },
     "sample": {
         "messages": [
@@ -8448,16 +8448,16 @@
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.02530047483742237,
-        "2": 0.9493933320045471,
-        "3": 0.02530047483742237
+        "1": 0.42432355880737305,
+        "2": 0.3744642436504364,
+        "3": 0.20043626427650452
     },
     "sample": {
         "messages": [
@@ -8487,21 +8487,21 @@
         "prompt_len": 161,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.09962064027786255,
-        "2": 0.7361025214195251,
-        "3": 0.16424667835235596
+        "1": 0.4509470462799072,
+        "2": 0.27351319789886475,
+        "3": 0.27351319789886475
     },
     "sample": {
         "messages": [
@@ -8531,7 +8531,7 @@
         "prompt_len": 114,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -8543,9 +8543,9 @@
         ]
     },
     "predict": {
-        "1": 0.7294971346855164,
-        "2": 0.1436464935541153,
-        "3": 0.1267675906419754
+        "1": 0.5937157273292542,
+        "2": 0.2474975287914276,
+        "3": 0.15011483430862427
     },
     "sample": {
         "messages": [
@@ -8587,9 +8587,9 @@
         ]
     },
     "predict": {
-        "1": 0.000977099291048944,
-        "2": 0.9456126093864441,
-        "3": 0.053347814828157425
+        "1": 0.07914919406175613,
+        "2": 0.5161175727844238,
+        "3": 0.4019527733325958
     },
     "sample": {
         "messages": [
@@ -8627,13 +8627,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.13583430647850037,
-        "2": 0.7816725373268127,
-        "3": 0.08238767832517624
+        "1": 0.4203770160675049,
+        "2": 0.4203770160675049,
+        "3": 0.1546480804681778
     },
     "sample": {
         "messages": [
@@ -8663,7 +8663,7 @@
         "prompt_len": 115,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -8675,9 +8675,9 @@
         ]
     },
     "predict": {
-        "1": 0.8978238701820374,
-        "2": 0.05739595741033554,
-        "3": 0.0447000153362751
+        "1": 0.6712876558303833,
+        "2": 0.19232714176177979,
+        "3": 0.13218437135219574
     },
     "sample": {
         "messages": [
@@ -8719,9 +8719,9 @@
         ]
     },
     "predict": {
-        "1": 0.6564561724662781,
-        "2": 0.12926387786865234,
-        "3": 0.21312011778354645
+        "1": 0.6324385404586792,
+        "2": 0.1811966896057129,
+        "3": 0.1811966896057129
     },
     "sample": {
         "messages": [
@@ -8763,9 +8763,9 @@
         ]
     },
     "predict": {
-        "1": 0.9665891528129578,
-        "2": 0.00032425453537143767,
-        "3": 0.033074863255023956
+        "1": 0.7968239784240723,
+        "2": 0.044953733682632446,
+        "3": 0.1569039225578308
     },
     "sample": {
         "messages": [
@@ -8807,9 +8807,9 @@
         ]
     },
     "predict": {
-        "1": 3.2874963835638482e-06,
-        "2": 0.9996157884597778,
-        "3": 0.0003799829282797873
+        "1": 0.14410072565078735,
+        "2": 0.6458145976066589,
+        "3": 0.20966529846191406
     },
     "sample": {
         "messages": [
@@ -8851,9 +8851,9 @@
         ]
     },
     "predict": {
-        "1": 0.0017215069383382797,
-        "2": 0.8917626738548279,
-        "3": 0.1065058633685112
+        "1": 0.1411750763654709,
+        "2": 0.5583582520484924,
+        "3": 0.2988676428794861
     },
     "sample": {
         "messages": [
@@ -8895,9 +8895,9 @@
         ]
     },
     "predict": {
-        "1": 0.8058937788009644,
-        "2": 0.15868988633155823,
-        "3": 0.03540850058197975
+        "1": 0.7890230417251587,
+        "2": 0.137111634016037,
+        "3": 0.07339057326316833
     },
     "sample": {
         "messages": [
@@ -8939,9 +8939,9 @@
         ]
     },
     "predict": {
-        "1": 0.9398670792579651,
-        "2": 4.2669900722103193e-05,
-        "3": 0.06008369103074074
+        "1": 0.7645320296287537,
+        "2": 0.015867354348301888,
+        "3": 0.2190420925617218
     },
     "sample": {
         "messages": [
@@ -8983,9 +8983,9 @@
         ]
     },
     "predict": {
-        "1": 0.7890399694442749,
-        "2": 0.13711456954479218,
-        "3": 0.07339214533567429
+        "1": 0.8434564471244812,
+        "2": 0.0784536674618721,
+        "3": 0.06923512369394302
     },
     "sample": {
         "messages": [
@@ -9027,9 +9027,9 @@
         ]
     },
     "predict": {
-        "1": 0.02442534640431404,
-        "2": 0.9165543913841248,
-        "3": 0.05859335884451866
+        "1": 0.35281944274902344,
+        "2": 0.39979681372642517,
+        "3": 0.2424890249967575
     },
     "sample": {
         "messages": [
@@ -9071,9 +9071,9 @@
         ]
     },
     "predict": {
-        "1": 0.9783619046211243,
-        "2": 0.005817552097141743,
-        "3": 0.01581374742090702
+        "1": 0.7395572662353516,
+        "2": 0.12851576507091522,
+        "3": 0.12851576507091522
     },
     "sample": {
         "messages": [
@@ -9115,9 +9115,9 @@
         ]
     },
     "predict": {
-        "1": 0.9993676543235779,
-        "2": 0.00029585754964500666,
-        "3": 0.0003352504863869399
+        "1": 0.9013769030570984,
+        "2": 0.05762309208512306,
+        "3": 0.039603736251592636
     },
     "sample": {
         "messages": [
@@ -9152,16 +9152,16 @@
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.2168908417224884,
-        "2": 0.7570233941078186,
-        "3": 0.025903915986418724
+        "1": 0.4002441465854645,
+        "2": 0.35321420431137085,
+        "3": 0.24276034533977509
     },
     "sample": {
         "messages": [
@@ -9191,7 +9191,7 @@
         "prompt_len": 97,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -9203,9 +9203,9 @@
         ]
     },
     "predict": {
-        "1": 0.9699215292930603,
-        "2": 0.0007805278874002397,
-        "3": 0.029289092868566513
+        "1": 0.788357138633728,
+        "2": 0.07332863658666611,
+        "3": 0.13699591159820557
     },
     "sample": {
         "messages": [
@@ -9243,13 +9243,13 @@
         "acc": false,
         "f1_macro": [
             "3",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.12808334827423096,
-        "2": 0.8352083563804626,
-        "3": 0.03669649362564087
+        "1": 0.7169543504714966,
+        "2": 0.14117667078971863,
+        "3": 0.14117667078971863
     },
     "sample": {
         "messages": [
@@ -9279,21 +9279,21 @@
         "prompt_len": 187,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": false,
+        "acc": true,
         "f1_macro": [
             "1",
-            "3"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.1864180713891983,
-        "2": 0.23936554789543152,
-        "3": 0.5742080807685852
+        "1": 0.6092379093170166,
+        "2": 0.1359393447637558,
+        "3": 0.25396811962127686
     },
     "sample": {
         "messages": [
@@ -9323,21 +9323,21 @@
         "prompt_len": 128,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "3"
+        "generated_token": "1"
     }
 }
 {
     "metric": {
-        "acc": true,
+        "acc": false,
         "f1_macro": [
             "2",
-            "2"
+            "1"
         ]
     },
     "predict": {
-        "1": 0.11355941742658615,
-        "2": 0.8390969038009644,
-        "3": 0.047338612377643585
+        "1": 0.6041527390480042,
+        "2": 0.2222554087638855,
+        "3": 0.17309266328811646
     },
     "sample": {
         "messages": [
@@ -9367,7 +9367,7 @@
         "prompt_len": 145,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": "2"
+        "generated_token": "1"
     }
 }
 {
@@ -9379,9 +9379,9 @@
         ]
     },
     "predict": {
-        "1": 0.9668984413146973,
-        "2": 1.2576736480696127e-05,
-        "3": 0.03308544680476189
+        "1": 0.792116105556488,
+        "2": 0.030713675543665886,
+        "3": 0.1767449975013733
     },
     "sample": {
         "messages": [
@@ -9423,9 +9423,9 @@
         ]
     },
     "predict": {
-        "1": 0.44953903555870056,
-        "2": 0.44953903555870056,
-        "3": 0.1003057211637497
+        "1": 0.5611218810081482,
+        "2": 0.30034691095352173,
+        "3": 0.1252032220363617
     },
     "sample": {
         "messages": [
@@ -9467,9 +9467,9 @@
         ]
     },
     "predict": {
-        "1": 0.8252539038658142,
-        "2": 0.08698112517595291,
-        "3": 0.08698112517595291
+        "1": 0.6750280857086182,
+        "2": 0.17067383229732513,
+        "3": 0.15061913430690765
     },
     "sample": {
         "messages": [
@@ -9511,9 +9511,9 @@
         ]
     },
     "predict": {
-        "1": 0.017948845401406288,
-        "2": 0.7632042765617371,
-        "3": 0.21866169571876526
+        "1": 0.3177637457847595,
+        "2": 0.3600735068321228,
+        "3": 0.3177637457847595
     },
     "sample": {
         "messages": [
@@ -9555,9 +9555,9 @@
         ]
     },
     "predict": {
-        "1": 0.43593066930770874,
-        "2": 0.26440533995628357,
-        "3": 0.2996104955673218
+        "1": 0.4433843195438385,
+        "2": 0.3453080654144287,
+        "3": 0.2094399333000183
     },
     "sample": {
         "messages": [
@@ -9599,9 +9599,9 @@
         ]
     },
     "predict": {
-        "1": 0.8239489197731018,
-        "2": 0.07663918286561966,
-        "3": 0.09840666502714157
+        "1": 0.8568756580352783,
+        "2": 0.07033663988113403,
+        "3": 0.07033663988113403
     },
     "sample": {
         "messages": [
@@ -9643,9 +9643,9 @@
         ]
     },
     "predict": {
-        "1": 0.8247462511062622,
-        "2": 0.09850189834833145,
-        "3": 0.07671334594488144
+        "1": 0.49714383482933044,
+        "2": 0.2661019265651703,
+        "3": 0.23483411967754364
     },
     "sample": {
         "messages": [

llmtf_eval/darumeru_RCB_params.jsonl

@@ -1,7 +1,7 @@
 {
     "custom_generation_config": null,
     "model_params": {
-        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
         "generation_config": {
             "bos_token_id": 145109,
             "do_sample": true,
@@ -36,7 +36,7 @@
         },
         "load_in_8bit": false,
         "torch_dtype": "auto",
-        "use_flash_attention_2": true,
+        "attn_implementation": "flash_attention_2",
         "device_map": "cuda:0",
         "use_fast_tokenizer": true,
         "leading_space": false,
@@ -47,7 +47,7 @@
     "task_params": {
         "max_len": 4000,
         "few_shot_count": 0,
-        "batch_size": 2,
+        "batch_size": 16,
         "max_sample_per_dataset": 10000000000000,
         "method": "calculate_tokens_proba"
     }

llmtf_eval/darumeru_RCB_total.jsonl

@@ -1,8 +1,8 @@
 {
     "task_name": "darumeru/RCB",
     "results": {
-        "acc": 0.4727272727272727,
-        "f1_macro": 0.39356669305497743
+        "acc": 0.4590909090909091,
+        "f1_macro": 0.36910715356478985
     },
-    "leaderboard_result": 0.4331469828911251
+    "leaderboard_result": 0.4140990313278495
 }

llmtf_eval/darumeru_RWSD.jsonl

@@ -3,8 +3,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.002472260966897011,
-        "Нет": 0.9973811507225037
+        "Да": 0.3177708387374878,
+        "Нет": 0.6727208495140076
     },
     "sample": {
         "messages": [
@@ -42,8 +42,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.7760170102119446,
-        "Нет": 0.22233259677886963
+        "Да": 0.5510627031326294,
+        "Нет": 0.42916804552078247
     },
     "sample": {
         "messages": [
@@ -81,8 +81,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 9.608079562895e-05,
-        "Нет": 0.9996789693832397
+        "Да": 0.1462704986333847,
+        "Нет": 0.8417286276817322
     },
     "sample": {
         "messages": [
@@ -120,8 +120,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0028004839550703764,
-        "Нет": 0.9970412850379944
+        "Да": 0.16274045407772064,
+        "Нет": 0.8264642357826233
     },
     "sample": {
         "messages": [
@@ -159,8 +159,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0330502949655056,
-        "Нет": 0.9658712148666382
+        "Да": 0.40234827995300293,
+        "Нет": 0.5854132771492004
     },
     "sample": {
         "messages": [
@@ -195,11 +195,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.00029593578074127436,
-        "Нет": 0.9996318817138672
+        "Да": 0.5870495438575745,
+        "Нет": 0.4034728705883026
     },
     "sample": {
         "messages": [
@@ -229,7 +229,7 @@
         "prompt_len": 172,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -237,8 +237,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.07581356167793274,
-        "Нет": 0.9235982894897461
+        "Да": 0.1812204122543335,
+        "Нет": 0.8121734857559204
     },
     "sample": {
         "messages": [
@@ -273,11 +273,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.4002552032470703,
-        "Нет": 0.5139378905296326
+        "Да": 0.555451512336731,
+        "Нет": 0.26237672567367554
     },
     "sample": {
         "messages": [
@@ -307,7 +307,7 @@
         "prompt_len": 183,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -315,8 +315,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.8652609586715698,
-        "Нет": 0.13269206881523132
+        "Да": 0.7572181224822998,
+        "Нет": 0.21694663166999817
     },
     "sample": {
         "messages": [
@@ -354,8 +354,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0850556269288063,
-        "Нет": 0.9144341945648193
+        "Да": 0.372804194688797,
+        "Нет": 0.6146501898765564
     },
     "sample": {
         "messages": [
@@ -393,8 +393,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.001324817189015448,
-        "Нет": 0.9985203146934509
+        "Да": 0.199379563331604,
+        "Нет": 0.7885614633560181
     },
     "sample": {
         "messages": [
@@ -432,8 +432,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.8660338521003723,
-        "Нет": 0.1328105926513672
+        "Да": 0.5521984696388245,
+        "Нет": 0.43005257844924927
     },
     "sample": {
         "messages": [
@@ -468,11 +468,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.015893418341875076,
-        "Нет": 0.9832910299301147
+        "Да": 0.8397411704063416,
+        "Нет": 0.1459251195192337
     },
     "sample": {
         "messages": [
@@ -502,7 +502,7 @@
         "prompt_len": 136,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -510,8 +510,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0533820241689682,
-        "Нет": 0.9462189674377441
+        "Да": 0.26557374000549316,
+        "Нет": 0.7219041585922241
     },
     "sample": {
         "messages": [
@@ -549,8 +549,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9315302968025208,
-        "Нет": 0.06747982650995255
+        "Да": 0.8436118364334106,
+        "Нет": 0.1465977430343628
     },
     "sample": {
         "messages": [
@@ -588,8 +588,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.00043051555985584855,
-        "Нет": 0.9994730353355408
+        "Да": 0.13194012641906738,
+        "Нет": 0.8603577017784119
     },
     "sample": {
         "messages": [
@@ -627,8 +627,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0015009645139798522,
-        "Нет": 0.9983540773391724
+        "Да": 0.3180491030216217,
+        "Нет": 0.6733099222183228
     },
     "sample": {
         "messages": [
@@ -663,11 +663,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.040754951536655426,
-        "Нет": 0.9275783896446228
+        "Да": 0.6367422342300415,
+        "Нет": 0.1824297159910202
     },
     "sample": {
         "messages": [
@@ -697,7 +697,7 @@
         "prompt_len": 157,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -705,8 +705,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.7037021517753601,
-        "Нет": 0.29334670305252075
+        "Да": 0.8310219645500183,
+        "Нет": 0.14440995454788208
     },
     "sample": {
         "messages": [
@@ -741,11 +741,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.40695691108703613,
-        "Нет": 0.5921187996864319
+        "Да": 0.6694736480712891,
+        "Нет": 0.3162369728088379
     },
     "sample": {
         "messages": [
@@ -775,7 +775,7 @@
         "prompt_len": 265,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -783,8 +783,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.001032112049870193,
-        "Нет": 0.9988526105880737
+        "Да": 0.1803991049528122,
+        "Нет": 0.8084926605224609
     },
     "sample": {
         "messages": [
@@ -822,8 +822,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.16420209407806396,
-        "Нет": 0.8338871002197266
+        "Да": 0.39615458250045776,
+        "Нет": 0.5764015316963196
     },
     "sample": {
         "messages": [
@@ -861,8 +861,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 4.5386968849925324e-05,
-        "Нет": 0.9997144341468811
+        "Да": 0.05950615927577019,
+        "Нет": 0.9308329820632935
     },
     "sample": {
         "messages": [
@@ -900,8 +900,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0004877749306615442,
-        "Нет": 0.9993436932563782
+        "Да": 0.290959894657135,
+        "Нет": 0.6979765295982361
     },
     "sample": {
         "messages": [
@@ -939,8 +939,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.009703288786113262,
-        "Нет": 0.9897623658180237
+        "Да": 0.344249963760376,
+        "Нет": 0.6431435942649841
     },
     "sample": {
         "messages": [
@@ -978,8 +978,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.00048780557699501514,
-        "Нет": 0.9994064569473267
+        "Да": 0.31753966212272644,
+        "Нет": 0.6722314357757568
     },
     "sample": {
         "messages": [
@@ -1017,8 +1017,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.000709479849319905,
-        "Нет": 0.9990220069885254
+        "Да": 0.047137245535850525,
+        "Нет": 0.9467768669128418
     },
     "sample": {
         "messages": [
@@ -1056,8 +1056,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.6570025682449341,
-        "Нет": 0.213297501206398
+        "Да": 0.6703915596008301,
+        "Нет": 0.11649657040834427
     },
     "sample": {
         "messages": [
@@ -1095,8 +1095,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.7527139782905579,
-        "Нет": 0.2443704456090927
+        "Да": 0.6857701539993286,
+        "Нет": 0.2858715355396271
     },
     "sample": {
         "messages": [
@@ -1134,8 +1134,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.10662366449832916,
-        "Нет": 0.892749011516571
+        "Да": 0.3724825084209442,
+        "Нет": 0.614119827747345
     },
     "sample": {
         "messages": [
@@ -1173,8 +1173,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.00029592280043289065,
-        "Нет": 0.9995880722999573
+        "Да": 0.1318102777004242,
+        "Нет": 0.8595109581947327
     },
     "sample": {
         "messages": [
@@ -1212,8 +1212,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.009701339527964592,
-        "Нет": 0.989563524723053
+        "Да": 0.315209299325943,
+        "Нет": 0.6672980785369873
     },
     "sample": {
         "messages": [
@@ -1251,8 +1251,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.00020335696171969175,
-        "Нет": 0.9994524121284485
+        "Да": 0.34368154406547546,
+        "Нет": 0.6420816779136658
     },
     "sample": {
         "messages": [
@@ -1287,11 +1287,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.0008039596723392606,
-        "Нет": 0.9990390539169312
+        "Да": 0.556214451789856,
+        "Нет": 0.4331802427768707
     },
     "sample": {
         "messages": [
@@ -1321,7 +1321,7 @@
         "prompt_len": 140,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -1329,8 +1329,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.7302190661430359,
-        "Нет": 0.2686326205730438
+        "Да": 0.5253308415412903,
+        "Нет": 0.46360284090042114
     },
     "sample": {
         "messages": [
@@ -1368,8 +1368,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.000803975504823029,
-        "Нет": 0.9990586638450623
+        "Да": 0.19996023178100586,
+        "Нет": 0.7908580899238586
     },
     "sample": {
         "messages": [
@@ -1407,8 +1407,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.014056859537959099,
-        "Нет": 0.985461950302124
+        "Да": 0.14682213962078094,
+        "Нет": 0.8449031114578247
     },
     "sample": {
         "messages": [
@@ -1446,8 +1446,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.7435306906700134,
-        "Нет": 0.21302512288093567
+        "Да": 0.6819260120391846,
+        "Нет": 0.10457674413919449
     },
     "sample": {
         "messages": [
@@ -1485,8 +1485,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9825862646102905,
-        "Нет": 0.01588202640414238
+        "Да": 0.7978547215461731,
+        "Нет": 0.17802545428276062
     },
     "sample": {
         "messages": [
@@ -1524,8 +1524,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.11913560330867767,
-        "Нет": 0.8802996873855591
+        "Да": 0.43058961629867554,
+        "Нет": 0.5528879761695862
     },
     "sample": {
         "messages": [
@@ -1563,8 +1563,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.029305994510650635,
-        "Нет": 0.970481276512146
+        "Да": 0.2914990186691284,
+        "Нет": 0.6992698311805725
     },
     "sample": {
         "messages": [
@@ -1602,8 +1602,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.222540944814682,
-        "Нет": 0.7767441868782043
+        "Да": 0.39979875087738037,
+        "Нет": 0.5817037224769592
     },
     "sample": {
         "messages": [
@@ -1638,11 +1638,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.0015003508888185024,
-        "Нет": 0.9979458451271057
+        "Да": 0.5226932168006897,
+        "Нет": 0.4612751305103302
     },
     "sample": {
         "messages": [
@@ -1672,16 +1672,16 @@
         "prompt_len": 139,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.03731626644730568,
-        "Нет": 0.9623991847038269
+        "Да": 0.644417941570282,
+        "Нет": 0.34493207931518555
     },
     "sample": {
         "messages": [
@@ -1711,7 +1711,7 @@
         "prompt_len": 129,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -1719,8 +1719,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.5304835438728333,
-        "Нет": 0.4681500494480133
+        "Да": 0.7686586380004883,
+        "Нет": 0.22022439539432526
     },
     "sample": {
         "messages": [
@@ -1758,8 +1758,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.001032113330438733,
-        "Нет": 0.9988539218902588
+        "Да": 0.2667292058467865,
+        "Нет": 0.7250450849533081
     },
     "sample": {
         "messages": [
@@ -1797,8 +1797,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.468466579914093,
-        "Нет": 0.5308421850204468
+        "Да": 0.40373584628105164,
+        "Нет": 0.5874322056770325
     },
     "sample": {
         "messages": [
@@ -1833,11 +1833,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.2149808406829834,
-        "Нет": 0.6621876358985901
+        "Да": 0.6286691427230835,
+        "Нет": 0.18011674284934998
     },
     "sample": {
         "messages": [
@@ -1867,7 +1867,7 @@
         "prompt_len": 147,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -1875,8 +1875,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9988813996315002,
-        "Нет": 0.0007093799649737775
+        "Да": 0.8896501064300537,
+        "Нет": 0.09376843273639679
     },
     "sample": {
         "messages": [
@@ -1914,8 +1914,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.060060080140829086,
-        "Нет": 0.939497709274292
+        "Да": 0.3719407021999359,
+        "Нет": 0.6132265329360962
     },
     "sample": {
         "messages": [
@@ -1953,8 +1953,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0004305129696149379,
-        "Нет": 0.9994670748710632
+        "Да": 0.180691197514534,
+        "Нет": 0.8098016977310181
     },
     "sample": {
         "messages": [
@@ -1992,8 +1992,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.06003950163722038,
-        "Нет": 0.9391758441925049
+        "Да": 0.26445913314819336,
+        "Нет": 0.7188743948936462
     },
     "sample": {
         "messages": [
@@ -2031,8 +2031,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0001794979179976508,
-        "Нет": 0.9996531009674072
+        "Да": 0.08422500640153885,
+        "Нет": 0.9055041670799255
     },
     "sample": {
         "messages": [
@@ -2070,8 +2070,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.000626234570518136,
-        "Нет": 0.999214768409729
+        "Да": 0.29040369391441345,
+        "Нет": 0.696642279624939
     },
     "sample": {
         "messages": [
@@ -2109,8 +2109,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.01590224727988243,
-        "Нет": 0.9838371872901917
+        "Да": 0.40303125977516174,
+        "Нет": 0.5864070057868958
     },
     "sample": {
         "messages": [
@@ -2148,8 +2148,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.00359282735735178,
-        "Нет": 0.9961913824081421
+        "Да": 0.34523308277130127,
+        "Нет": 0.6449803113937378
     },
     "sample": {
         "messages": [
@@ -2187,8 +2187,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.00857413001358509,
-        "Нет": 0.9910346865653992
+        "Да": 0.22018398344516754,
+        "Нет": 0.7685176134109497
     },
     "sample": {
         "messages": [
@@ -2226,8 +2226,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.8524937033653259,
-        "Нет": 0.1018158569931984
+        "Да": 0.7261016964912415,
+        "Нет": 0.06753797829151154
     },
     "sample": {
         "messages": [
@@ -2262,11 +2262,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.07573917508125305,
-        "Нет": 0.922692060470581
+        "Да": 0.8162997364997864,
+        "Нет": 0.16073894500732422
     },
     "sample": {
         "messages": [
@@ -2296,7 +2296,7 @@
         "prompt_len": 86,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -2304,8 +2304,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.02594723552465439,
-        "Нет": 0.9736628532409668
+        "Да": 0.11767112463712692,
+        "Нет": 0.8694785237312317
     },
     "sample": {
         "messages": [
@@ -2343,8 +2343,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.008576461113989353,
-        "Нет": 0.9913041591644287
+        "Да": 0.3737662136554718,
+        "Нет": 0.6162363290786743
     },
     "sample": {
         "messages": [
@@ -2382,8 +2382,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.49879026412963867,
-        "Нет": 0.49879026412963867
+        "Да": 0.48837897181510925,
+        "Нет": 0.48837897181510925
     },
     "sample": {
         "messages": [
@@ -2421,8 +2421,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0003351966734044254,
-        "Нет": 0.9992071986198425
+        "Да": 0.13113625347614288,
+        "Нет": 0.8551157712936401
     },
     "sample": {
         "messages": [
@@ -2460,8 +2460,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 1.8922029994428158e-05,
-        "Нет": 0.999816358089447
+        "Да": 0.14641688764095306,
+        "Нет": 0.8425710797309875
     },
     "sample": {
         "messages": [
@@ -2496,11 +2496,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.053379327058792114,
-        "Нет": 0.9461711645126343
+        "Да": 0.5243616104125977,
+        "Нет": 0.4627474844455719
     },
     "sample": {
         "messages": [
@@ -2530,16 +2530,16 @@
         "prompt_len": 204,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.0017005337867885828,
-        "Нет": 0.9981884360313416
+        "Да": 0.6734837889671326,
+        "Нет": 0.3181312084197998
     },
     "sample": {
         "messages": [
@@ -2569,7 +2569,7 @@
         "prompt_len": 157,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -2577,8 +2577,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.017960019409656525,
-        "Нет": 0.9805837869644165
+        "Да": 0.1466711312532425,
+        "Нет": 0.8440341353416443
     },
     "sample": {
         "messages": [
@@ -2613,11 +2613,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.4208765923976898,
-        "Нет": 0.5404162406921387
+        "Да": 0.6537999510765076,
+        "Нет": 0.16530652344226837
     },
     "sample": {
         "messages": [
@@ -2647,7 +2647,7 @@
         "prompt_len": 168,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -2655,8 +2655,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.8503593802452087,
-        "Нет": 0.14777028560638428
+        "Да": 0.6613848209381104,
+        "Нет": 0.31241607666015625
     },
     "sample": {
         "messages": [
@@ -2694,8 +2694,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.05337907001376152,
-        "Нет": 0.9461665749549866
+        "Да": 0.2903401851654053,
+        "Нет": 0.6964899301528931
     },
     "sample": {
         "messages": [
@@ -2733,8 +2733,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0019263619324192405,
-        "Нет": 0.997880220413208
+        "Да": 0.2198316901922226,
+        "Нет": 0.7672879695892334
     },
     "sample": {
         "messages": [
@@ -2769,11 +2769,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.4366450011730194,
-        "Нет": 0.5606632828712463
+        "Да": 0.5178764462471008,
+        "Нет": 0.45702433586120605
     },
     "sample": {
         "messages": [
@@ -2803,7 +2803,7 @@
         "prompt_len": 115,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -2811,8 +2811,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.9451661705970764,
-        "Нет": 0.05332263186573982
+        "Да": 0.8425301909446716,
+        "Нет": 0.1464097797870636
     },
     "sample": {
         "messages": [
@@ -2847,11 +2847,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.0019265249138697982,
-        "Нет": 0.99796462059021
+        "Да": 0.524517834186554,
+        "Нет": 0.4628853499889374
     },
     "sample": {
         "messages": [
@@ -2881,7 +2881,7 @@
         "prompt_len": 126,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -2889,8 +2889,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9129877090454102,
-        "Нет": 0.08492108434438705
+        "Да": 0.721961498260498,
+        "Нет": 0.26559484004974365
     },
     "sample": {
         "messages": [
@@ -2928,8 +2928,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.002800603397190571,
-        "Нет": 0.997083842754364
+        "Да": 0.05969366058707237,
+        "Нет": 0.9337659478187561
     },
     "sample": {
         "messages": [
@@ -2967,8 +2967,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0019256919622421265,
-        "Нет": 0.9975330829620361
+        "Да": 0.11805005371570587,
+        "Нет": 0.8722785115242004
     },
     "sample": {
         "messages": [
@@ -3003,11 +3003,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.008415551856160164,
-        "Нет": 0.972705602645874
+        "Да": 0.7121773958206177,
+        "Нет": 0.12375786900520325
     },
     "sample": {
         "messages": [
@@ -3037,7 +3037,7 @@
         "prompt_len": 185,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -3045,8 +3045,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.9229567646980286,
-        "Нет": 0.07576090097427368
+        "Да": 0.7993603944778442,
+        "Нет": 0.1783614158630371
     },
     "sample": {
         "messages": [
@@ -3081,11 +3081,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.3771263360977173,
-        "Нет": 0.6217762231826782
+        "Да": 0.4931306540966034,
+        "Нет": 0.4931306540966034
     },
     "sample": {
         "messages": [
@@ -3115,7 +3115,7 @@
         "prompt_len": 256,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -3123,8 +3123,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.00043048226507380605,
-        "Нет": 0.9993957281112671
+        "Да": 0.24220842123031616,
+        "Нет": 0.746054470539093
     },
     "sample": {
         "messages": [
@@ -3162,8 +3162,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.7047868371009827,
-        "Нет": 0.2937988340854645
+        "Да": 0.6129973530769348,
+        "Нет": 0.3718017041683197
     },
     "sample": {
         "messages": [
@@ -3198,11 +3198,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.010979406535625458,
-        "Нет": 0.9883346557617188
+        "Да": 0.6393494009971619,
+        "Нет": 0.3422190845012665
     },
     "sample": {
         "messages": [
@@ -3232,7 +3232,7 @@
         "prompt_len": 109,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -3240,8 +3240,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0010320731671527028,
-        "Нет": 0.9988150596618652
+        "Да": 0.29139018058776855,
+        "Нет": 0.6990087032318115
     },
     "sample": {
         "messages": [
@@ -3279,8 +3279,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9235284924507141,
-        "Нет": 0.0758078396320343
+        "Да": 0.859817624092102,
+        "Нет": 0.13185730576515198
     },
     "sample": {
         "messages": [
@@ -3318,8 +3318,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 6.60449659335427e-05,
-        "Нет": 0.9998252987861633
+        "Да": 0.0671381726861,
+        "Нет": 0.9268138408660889
     },
     "sample": {
         "messages": [
@@ -3357,8 +3357,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.08504106849431992,
-        "Нет": 0.9142776131629944
+        "Да": 0.2661048471927643,
+        "Нет": 0.7233479022979736
     },
     "sample": {
         "messages": [
@@ -3396,8 +3396,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9325262308120728,
-        "Нет": 0.024850983172655106
+        "Да": 0.7852900624275208,
+        "Нет": 0.044303033500909805
     },
     "sample": {
         "messages": [
@@ -3432,11 +3432,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.18190623819828033,
-        "Нет": 0.8152471780776978
+        "Да": 0.661439061164856,
+        "Нет": 0.3124416768550873
     },
     "sample": {
         "messages": [
@@ -3466,7 +3466,7 @@
         "prompt_len": 134,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -3474,8 +3474,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.11914443224668503,
-        "Нет": 0.8803648948669434
+        "Да": 0.4621798098087311,
+        "Нет": 0.5237182974815369
     },
     "sample": {
         "messages": [
@@ -3513,8 +3513,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0002611565578263253,
-        "Нет": 0.999609649181366
+        "Да": 0.13172535598278046,
+        "Нет": 0.8589572310447693
     },
     "sample": {
         "messages": [
@@ -3552,8 +3552,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.025938672944903374,
-        "Нет": 0.9733415842056274
+        "Да": 0.19765295088291168,
+        "Нет": 0.7817325592041016
     },
     "sample": {
         "messages": [
@@ -3591,8 +3591,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0002611354284454137,
-        "Нет": 0.9995287656784058
+        "Да": 0.26451557874679565,
+        "Нет": 0.7190278172492981
     },
     "sample": {
         "messages": [
@@ -3630,8 +3630,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0019264277070760727,
-        "Нет": 0.9979142546653748
+        "Да": 0.43345609307289124,
+        "Нет": 0.5565686225891113
     },
     "sample": {
         "messages": [
@@ -3669,8 +3669,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.9939780235290527,
-        "Нет": 0.005215918179601431
+        "Да": 0.8087101578712463,
+        "Нет": 0.18044762313365936
     },
     "sample": {
         "messages": [
@@ -3708,8 +3708,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0021823798306286335,
-        "Нет": 0.9976633787155151
+        "Да": 0.18087710440158844,
+        "Нет": 0.8106349110603333
     },
     "sample": {
         "messages": [
@@ -3747,8 +3747,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0008039618842303753,
-        "Нет": 0.999041736125946
+        "Да": 0.09458442777395248,
+        "Нет": 0.8973921537399292
     },
     "sample": {
         "messages": [
@@ -3786,8 +3786,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9750506281852722,
-        "Нет": 0.002738716546446085
+        "Да": 0.8824678659439087,
+        "Нет": 0.018315035849809647
     },
     "sample": {
         "messages": [
@@ -3822,11 +3822,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.4672533869743347,
-        "Нет": 0.5294674634933472
+        "Да": 0.8161836266517639,
+        "Нет": 0.16071607172489166
     },
     "sample": {
         "messages": [
@@ -3856,7 +3856,7 @@
         "prompt_len": 127,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -3864,8 +3864,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.24489443004131317,
-        "Нет": 0.7543279528617859
+        "Да": 0.31640368700027466,
+        "Нет": 0.6698265671730042
     },
     "sample": {
         "messages": [
@@ -3903,8 +3903,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.004069507587701082,
-        "Нет": 0.9957756400108337
+        "Да": 0.31690847873687744,
+        "Нет": 0.6708952188491821
     },
     "sample": {
         "messages": [
@@ -3942,8 +3942,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.7040597200393677,
-        "Нет": 0.2934957444667816
+        "Да": 0.5515921115875244,
+        "Нет": 0.42958036065101624
     },
     "sample": {
         "messages": [
@@ -3981,8 +3981,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 2.7526933990884572e-05,
-        "Нет": 0.9996544122695923
+        "Да": 0.02903013676404953,
+        "Нет": 0.9613460898399353
     },
     "sample": {
         "messages": [
@@ -4020,8 +4020,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.002800086047500372,
-        "Нет": 0.9968996047973633
+        "Да": 0.4634995758533478,
+        "Нет": 0.5252138376235962
     },
     "sample": {
         "messages": [
@@ -4059,8 +4059,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.816844642162323,
-        "Нет": 0.18226267397403717
+        "Да": 0.6726118922233582,
+        "Нет": 0.31771937012672424
     },
     "sample": {
         "messages": [
@@ -4098,8 +4098,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0011693844571709633,
-        "Нет": 0.9987230896949768
+        "Да": 0.10598231852054596,
+        "Нет": 0.8873790502548218
     },
     "sample": {
         "messages": [
@@ -4137,8 +4137,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.000626144465059042,
-        "Нет": 0.9990710020065308
+        "Да": 0.09457684308290482,
+        "Нет": 0.8973201513290405
     },
     "sample": {
         "messages": [
@@ -4176,8 +4176,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.688129723072052,
-        "Нет": 0.25314879417419434
+        "Да": 0.759200930595398,
+        "Нет": 0.07061668485403061
     },
     "sample": {
         "messages": [
@@ -4215,8 +4215,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.997204065322876,
-        "Нет": 0.0021813749335706234
+        "Да": 0.9164130091667175,
+        "Нет": 0.06638473272323608
     },
     "sample": {
         "messages": [
@@ -4254,8 +4254,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.13287986814975739,
-        "Нет": 0.866485595703125
+        "Да": 0.34382012486457825,
+        "Нет": 0.6423404812812805
     },
     "sample": {
         "messages": [
@@ -4293,8 +4293,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.04741421714425087,
-        "Нет": 0.9523400068283081
+        "Да": 0.40259450674057007,
+        "Нет": 0.5857715606689453
     },
     "sample": {
         "messages": [
@@ -4329,11 +4329,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.20164957642555237,
-        "Нет": 0.7975395321846008
+        "Да": 0.5218465924263,
+        "Нет": 0.46052801609039307
     },
     "sample": {
         "messages": [
@@ -4363,7 +4363,7 @@
         "prompt_len": 117,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -4371,8 +4371,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.00023044571571517736,
-        "Нет": 0.9995049238204956
+        "Да": 0.3146660625934601,
+        "Нет": 0.6661480665206909
     },
     "sample": {
         "messages": [
@@ -4410,8 +4410,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.01798146963119507,
-        "Нет": 0.9817549586296082
+        "Да": 0.402546763420105,
+        "Нет": 0.5857020616531372
     },
     "sample": {
         "messages": [
@@ -4446,11 +4446,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.07579170912504196,
-        "Нет": 0.9233320355415344
+        "Да": 0.4943414032459259,
+        "Нет": 0.4943414032459259
     },
     "sample": {
         "messages": [
@@ -4480,7 +4480,7 @@
         "prompt_len": 111,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -4488,8 +4488,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.00317213824018836,
-        "Нет": 0.9966561794281006
+        "Да": 0.16312016546726227,
+        "Нет": 0.8283925652503967
     },
     "sample": {
         "messages": [
@@ -4527,8 +4527,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.16443610191345215,
-        "Нет": 0.8350754976272583
+        "Да": 0.4651126265525818,
+        "Нет": 0.5270416736602783
     },
     "sample": {
         "messages": [
@@ -4566,8 +4566,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9623715877532959,
-        "Нет": 0.005050062667578459
+        "Да": 0.8558553457260132,
+        "Нет": 0.025844592601060867
     },
     "sample": {
         "messages": [
@@ -4602,11 +4602,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.05997878313064575,
-        "Нет": 0.938226044178009
+        "Да": 0.5476301908493042,
+        "Нет": 0.4264948070049286
     },
     "sample": {
         "messages": [
@@ -4636,7 +4636,7 @@
         "prompt_len": 106,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -4644,8 +4644,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0850648358464241,
-        "Нет": 0.9145331978797913
+        "Да": 0.3729587197303772,
+        "Нет": 0.6149049401283264
     },
     "sample": {
         "messages": [
@@ -4680,11 +4680,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.007576117757707834,
-        "Нет": 0.9922756552696228
+        "Да": 0.5560380220413208,
+        "Нет": 0.43304288387298584
     },
     "sample": {
         "messages": [
@@ -4714,7 +4714,7 @@
         "prompt_len": 104,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -4722,8 +4722,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0259372740983963,
-        "Нет": 0.9732890725135803
+        "Да": 0.26346248388290405,
+        "Нет": 0.7161652445793152
     },
     "sample": {
         "messages": [
@@ -4761,8 +4761,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.00020336236048024148,
-        "Нет": 0.9994789958000183
+        "Да": 0.24148209393024445,
+        "Нет": 0.7438172698020935
     },
     "sample": {
         "messages": [
@@ -4797,11 +4797,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.047405000776052475,
-        "Нет": 0.9521549344062805
+        "Да": 0.64447420835495,
+        "Нет": 0.3449622094631195
     },
     "sample": {
         "messages": [
@@ -4831,7 +4831,7 @@
         "prompt_len": 105,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -4839,8 +4839,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.007575278170406818,
-        "Нет": 0.9921656847000122
+        "Да": 0.14658458530902863,
+        "Нет": 0.8435360789299011
     },
     "sample": {
         "messages": [
@@ -4878,8 +4878,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.00043050554813817143,
-        "Нет": 0.9994497895240784
+        "Да": 0.16303664445877075,
+        "Нет": 0.8279684782028198
     },
     "sample": {
         "messages": [
@@ -4917,8 +4917,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.005217640660703182,
-        "Нет": 0.9943062663078308
+        "Да": 0.14677007496356964,
+        "Нет": 0.8446035385131836
     },
     "sample": {
         "messages": [
@@ -4956,8 +4956,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.8910460472106934,
-        "Нет": 0.06454716622829437
+        "Да": 0.7619085907936096,
+        "Нет": 0.10311310738325119
     },
     "sample": {
         "messages": [
@@ -4995,8 +4995,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0040669688023626804,
-        "Нет": 0.9951545000076294
+        "Да": 0.26235565543174744,
+        "Нет": 0.713156521320343
     },
     "sample": {
         "messages": [
@@ -5031,11 +5031,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.26880720257759094,
-        "Нет": 0.7306936383247375
+        "Да": 0.4940703511238098,
+        "Нет": 0.4940703511238098
     },
     "sample": {
         "messages": [
@@ -5065,16 +5065,16 @@
         "prompt_len": 228,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.022972356528043747,
-        "Нет": 0.9768094420433044
+        "Да": 0.556427538394928,
+        "Нет": 0.4333462119102478
     },
     "sample": {
         "messages": [
@@ -5104,7 +5104,7 @@
         "prompt_len": 101,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -5112,8 +5112,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.02594570256769657,
-        "Нет": 0.9736053347587585
+        "Да": 0.2407470941543579,
+        "Нет": 0.7415533065795898
     },
     "sample": {
         "messages": [
@@ -5151,8 +5151,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0007094802567735314,
-        "Нет": 0.9990226030349731
+        "Да": 0.2656715512275696,
+        "Нет": 0.7221701145172119
     },
     "sample": {
         "messages": [
@@ -5190,8 +5190,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0035929649602621794,
-        "Нет": 0.9962295889854431
+        "Да": 0.46243879199028015,
+        "Нет": 0.5240117907524109
     },
     "sample": {
         "messages": [
@@ -5229,8 +5229,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9318119287490845,
-        "Нет": 0.06750023365020752
+        "Да": 0.7232219576835632,
+        "Нет": 0.2660585045814514
     },
     "sample": {
         "messages": [
@@ -5268,8 +5268,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0024723317474126816,
-        "Нет": 0.9974097609519958
+        "Да": 0.4041346609592438,
+        "Нет": 0.5880124568939209
     },
     "sample": {
         "messages": [
@@ -5307,8 +5307,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.004607855807989836,
-        "Нет": 0.9950196743011475
+        "Да": 0.1467008739709854,
+        "Нет": 0.8442053198814392
     },
     "sample": {
         "messages": [
@@ -5346,8 +5346,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.8451831340789795,
-        "Нет": 0.0693768560886383
+        "Да": 0.6610932946205139,
+        "Нет": 0.1894063949584961
     },
     "sample": {
         "messages": [
@@ -5382,11 +5382,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.025922434404492378,
-        "Нет": 0.9727321863174438
+        "Да": 0.7113779783248901,
+        "Нет": 0.2617013454437256
     },
     "sample": {
         "messages": [
@@ -5416,16 +5416,16 @@
         "prompt_len": 78,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.13289140164852142,
-        "Нет": 0.8665607571601868
+        "Да": 0.4932982623577118,
+        "Нет": 0.4932982623577118
     },
     "sample": {
         "messages": [
@@ -5455,7 +5455,7 @@
         "prompt_len": 224,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -5463,8 +5463,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0004305104666855186,
-        "Нет": 0.9994612336158752
+        "Да": 0.291109561920166,
+        "Нет": 0.698335587978363
     },
     "sample": {
         "messages": [
@@ -5502,8 +5502,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.468307226896286,
-        "Нет": 0.5306615829467773
+        "Да": 0.4291605055332184,
+        "Нет": 0.551052987575531
     },
     "sample": {
         "messages": [
@@ -5541,8 +5541,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0002304443478351459,
-        "Нет": 0.9994989633560181
+        "Да": 0.2900485098361969,
+        "Нет": 0.69579017162323
     },
     "sample": {
         "messages": [
@@ -5580,8 +5580,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0003799199184868485,
-        "Нет": 0.9994500279426575
+        "Да": 0.2201392948627472,
+        "Нет": 0.7683616280555725
     },
     "sample": {
         "messages": [
@@ -5616,11 +5616,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.2447185069322586,
-        "Нет": 0.7537860870361328
+        "Да": 0.5840882062911987,
+        "Нет": 0.40143755078315735
     },
     "sample": {
         "messages": [
@@ -5650,7 +5650,7 @@
         "prompt_len": 105,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -5658,8 +5658,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0015010004863142967,
-        "Нет": 0.9983779191970825
+        "Да": 0.4643440246582031,
+        "Нет": 0.5261707305908203
     },
     "sample": {
         "messages": [
@@ -5697,8 +5697,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.012427574954926968,
-        "Нет": 0.9872445464134216
+        "Да": 0.24299320578575134,
+        "Нет": 0.748471736907959
     },
     "sample": {
         "messages": [
@@ -5736,8 +5736,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.7317188382148743,
-        "Нет": 0.23755432665348053
+        "Да": 0.6578083634376526,
+        "Нет": 0.14677689969539642
     },
     "sample": {
         "messages": [
@@ -5772,11 +5772,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.4057359993457794,
-        "Нет": 0.5903423428535461
+        "Да": 0.7556150555610657,
+        "Нет": 0.2164873480796814
     },
     "sample": {
         "messages": [
@@ -5806,7 +5806,7 @@
         "prompt_len": 128,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -5814,8 +5814,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.16444909572601318,
-        "Нет": 0.8351414799690247
+        "Да": 0.402601957321167,
+        "Нет": 0.5857823491096497
     },
     "sample": {
         "messages": [
@@ -5853,8 +5853,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.002182485768571496,
-        "Нет": 0.9977117776870728
+        "Да": 0.19968147575855255,
+        "Нет": 0.7897555232048035
     },
     "sample": {
         "messages": [
@@ -5892,8 +5892,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.4990127682685852,
-        "Нет": 0.4990127682685852
+        "Да": 0.48904749751091003,
+        "Нет": 0.48904749751091003
     },
     "sample": {
         "messages": [
@@ -5931,8 +5931,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.09520400315523148,
-        "Нет": 0.9032704830169678
+        "Да": 0.46068769693374634,
+        "Нет": 0.5220275521278381
     },
     "sample": {
         "messages": [
@@ -5970,8 +5970,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0006262222304940224,
-        "Нет": 0.9991951584815979
+        "Да": 0.3446996510028839,
+        "Нет": 0.6439837217330933
     },
     "sample": {
         "messages": [
@@ -6006,11 +6006,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.4373700022697449,
-        "Нет": 0.5615941882133484
+        "Да": 0.5551593899726868,
+        "Нет": 0.43235859274864197
     },
     "sample": {
         "messages": [
@@ -6040,16 +6040,16 @@
         "prompt_len": 206,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.020326819270849228,
-        "Нет": 0.9794010519981384
+        "Да": 0.6718888282775879,
+        "Нет": 0.31737783551216125
     },
     "sample": {
         "messages": [
@@ -6079,7 +6079,7 @@
         "prompt_len": 174,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -6087,8 +6087,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.2687530517578125,
-        "Нет": 0.7305464744567871
+        "Да": 0.29172345995903015,
+        "Нет": 0.6998081803321838
     },
     "sample": {
         "messages": [
@@ -6126,8 +6126,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.6943596601486206,
-        "Нет": 0.17556162178516388
+        "Да": 0.7152285575866699,
+        "Нет": 0.14083683490753174
     },
     "sample": {
         "messages": [
@@ -6162,11 +6162,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.16413532197475433,
-        "Нет": 0.8335480093955994
+        "Да": 0.6622115969657898,
+        "Нет": 0.312806636095047
     },
     "sample": {
         "messages": [
@@ -6196,7 +6196,7 @@
         "prompt_len": 105,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -6204,8 +6204,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.08501418679952621,
-        "Нет": 0.9139886498451233
+        "Да": 0.3157346844673157,
+        "Нет": 0.6684103012084961
     },
     "sample": {
         "messages": [
@@ -6243,8 +6243,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0013248433824628592,
-        "Нет": 0.9985400438308716
+        "Да": 0.316948264837265,
+        "Нет": 0.6709794402122498
     },
     "sample": {
         "messages": [
@@ -6279,11 +6279,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.22250330448150635,
-        "Нет": 0.7766128182411194
+        "Да": 0.4906494915485382,
+        "Нет": 0.4906494915485382
     },
     "sample": {
         "messages": [
@@ -6313,7 +6313,7 @@
         "prompt_len": 114,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -6321,8 +6321,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 1.8911578081315383e-05,
-        "Нет": 0.9992640614509583
+        "Да": 0.032708365470170975,
+        "Нет": 0.9558786153793335
     },
     "sample": {
         "messages": [
@@ -6360,8 +6360,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.001169274328276515,
-        "Нет": 0.998629093170166
+        "Да": 0.4026472568511963,
+        "Нет": 0.5858482718467712
     },
     "sample": {
         "messages": [
@@ -6399,8 +6399,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.9763138890266418,
-        "Нет": 0.02296070195734501
+        "Да": 0.7218838930130005,
+        "Нет": 0.26556628942489624
     },
     "sample": {
         "messages": [
@@ -6438,8 +6438,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.00017950989422388375,
-        "Нет": 0.9997197985649109
+        "Да": 0.16291382908821106,
+        "Нет": 0.8273447155952454
     },
     "sample": {
         "messages": [
@@ -6477,8 +6477,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0019262604182586074,
-        "Нет": 0.9978275895118713
+        "Да": 0.06710854917764664,
+        "Нет": 0.9264049530029297
     },
     "sample": {
         "messages": [
@@ -6516,8 +6516,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.7125749588012695,
-        "Нет": 0.2313392162322998
+        "Да": 0.7096846699714661,
+        "Нет": 0.1583520472049713
     },
     "sample": {
         "messages": [
@@ -6552,11 +6552,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.13258744776248932,
-        "Нет": 0.8645787239074707
+        "Да": 0.6868926286697388,
+        "Нет": 0.2863394320011139
     },
     "sample": {
         "messages": [
@@ -6586,7 +6586,7 @@
         "prompt_len": 104,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -6594,8 +6594,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.26871126890182495,
-        "Нет": 0.7304328680038452
+        "Да": 0.431057870388031,
+        "Нет": 0.5534892678260803
     },
     "sample": {
         "messages": [
@@ -6633,8 +6633,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0003799440455622971,
-        "Нет": 0.9995135068893433
+        "Да": 0.11795960366725922,
+        "Нет": 0.871610164642334
     },
     "sample": {
         "messages": [
@@ -6672,8 +6672,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.11908745020627975,
-        "Нет": 0.8799439072608948
+        "Да": 0.3427065312862396,
+        "Нет": 0.6402600407600403
     },
     "sample": {
         "messages": [
@@ -6711,8 +6711,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0001233512011822313,
-        "Нет": 0.9995251893997192
+        "Да": 0.0841980129480362,
+        "Нет": 0.9052139520645142
     },
     "sample": {
         "messages": [
@@ -6750,8 +6750,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0075753238052129745,
-        "Нет": 0.9921716451644897
+        "Да": 0.4628131091594696,
+        "Нет": 0.5244359374046326
     },
     "sample": {
         "messages": [
@@ -6789,8 +6789,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.8166061639785767,
-        "Нет": 0.18220946192741394
+        "Да": 0.49528276920318604,
+        "Нет": 0.49528276920318604
     },
     "sample": {
         "messages": [
@@ -6828,8 +6828,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0019264371367171407,
-        "Нет": 0.9979191422462463
+        "Да": 0.4035145938396454,
+        "Нет": 0.5871102809906006
     },
     "sample": {
         "messages": [
@@ -6867,8 +6867,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.010980246588587761,
-        "Нет": 0.988410234451294
+        "Да": 0.11797474324703217,
+        "Нет": 0.8717220425605774
     },
     "sample": {
         "messages": [
@@ -6906,8 +6906,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.8517078161239624,
-        "Нет": 0.06169750913977623
+        "Да": 0.7332377433776855,
+        "Нет": 0.12741760909557343
     },
     "sample": {
         "messages": [
@@ -6942,11 +6942,11 @@
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.20117761194705963,
-        "Нет": 0.7956728935241699
+        "Да": 0.8314090371131897,
+        "Нет": 0.1444772183895111
     },
     "sample": {
         "messages": [
@@ -6976,7 +6976,7 @@
         "prompt_len": 129,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -6984,8 +6984,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.40705427527427673,
-        "Нет": 0.592260479927063
+        "Да": 0.4627135396003723,
+        "Нет": 0.5243231058120728
     },
     "sample": {
         "messages": [
@@ -7023,8 +7023,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 9.60883335210383e-05,
-        "Нет": 0.9997573494911194
+        "Да": 0.06682266294956207,
+        "Нет": 0.9224584102630615
     },
     "sample": {
         "messages": [
@@ -7059,11 +7059,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.43702271580696106,
-        "Нет": 0.5611482858657837
+        "Да": 0.5513796210289001,
+        "Нет": 0.42941489815711975
     },
     "sample": {
         "messages": [
@@ -7093,7 +7093,7 @@
         "prompt_len": 143,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -7101,8 +7101,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.005216446705162525,
-        "Нет": 0.9940787553787231
+        "Да": 0.4604188799858093,
+        "Нет": 0.5217229723930359
     },
     "sample": {
         "messages": [
@@ -7140,8 +7140,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0007095822365954518,
-        "Нет": 0.9991662502288818
+        "Да": 0.2908935546875,
+        "Нет": 0.6978173851966858
     },
     "sample": {
         "messages": [
@@ -7176,11 +7176,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.2448546439409256,
-        "Нет": 0.7542054057121277
+        "Да": 0.5554031729698181,
+        "Нет": 0.4325484335422516
     },
     "sample": {
         "messages": [
@@ -7210,7 +7210,7 @@
         "prompt_len": 141,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -7218,8 +7218,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0011693498818203807,
-        "Нет": 0.9986935257911682
+        "Да": 0.3448418080806732,
+        "Нет": 0.6442492604255676
     },
     "sample": {
         "messages": [
@@ -7257,8 +7257,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.06752399355173111,
-        "Нет": 0.9321399331092834
+        "Да": 0.24322527647018433,
+        "Нет": 0.749186635017395
     },
     "sample": {
         "messages": [
@@ -7296,8 +7296,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.7586944103240967,
-        "Нет": 0.11634956300258636
+        "Да": 0.6574149131774902,
+        "Нет": 0.1142415776848793
     },
     "sample": {
         "messages": [
@@ -7335,8 +7335,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.7757806777954102,
-        "Нет": 0.22226488590240479
+        "Да": 0.8883598446846008,
+        "Нет": 0.09363243728876114
     },
     "sample": {
         "messages": [
@@ -7371,11 +7371,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.37726110219955444,
-        "Нет": 0.6219983696937561
+        "Да": 0.5544126033782959,
+        "Нет": 0.4317769706249237
     },
     "sample": {
         "messages": [
@@ -7405,7 +7405,7 @@
         "prompt_len": 256,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -7413,8 +7413,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.0040694172494113445,
-        "Нет": 0.9957535266876221
+        "Да": 0.24236559867858887,
+        "Нет": 0.7465386390686035
     },
     "sample": {
         "messages": [
@@ -7452,8 +7452,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.40673938393592834,
-        "Нет": 0.5918022990226746
+        "Да": 0.36993473768234253,
+        "Нет": 0.6099192500114441
     },
     "sample": {
         "messages": [
@@ -7491,8 +7491,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.9224575757980347,
-        "Нет": 0.0757199302315712
+        "Да": 0.8060645461082458,
+        "Нет": 0.1798573136329651
     },
     "sample": {
         "messages": [
@@ -7530,8 +7530,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.005219148471951485,
-        "Нет": 0.9945936799049377
+        "Да": 0.4632495641708374,
+        "Нет": 0.5249305367469788
     },
     "sample": {
         "messages": [
@@ -7566,11 +7566,11 @@
 }
 {
     "metric": {
-        "acc": true
+        "acc": false
     },
     "predict": {
-        "Да": 0.1479138880968094,
-        "Нет": 0.8511857390403748
+        "Да": 0.4942798912525177,
+        "Нет": 0.4942798912525177
     },
     "sample": {
         "messages": [
@@ -7600,16 +7600,16 @@
         "prompt_len": 115,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
     "metric": {
-        "acc": false
+        "acc": true
     },
     "predict": {
-        "Да": 0.015902651473879814,
-        "Нет": 0.9838622212409973
+        "Да": 0.5872390866279602,
+        "Нет": 0.4036031663417816
     },
     "sample": {
         "messages": [
@@ -7639,7 +7639,7 @@
         "prompt_len": 175,
         "generated_len": 1,
         "generated_cumulative_logprob": "TODO: calculate for hf model",
-        "generated_token": " Нет"
+        "generated_token": " Да"
     }
 }
 {
@@ -7647,8 +7647,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.05337413400411606,
-        "Нет": 0.9460790753364563
+        "Да": 0.266113817691803,
+        "Нет": 0.723372220993042
     },
     "sample": {
         "messages": [
@@ -7686,8 +7686,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.03931839391589165,
-        "Нет": 0.8948825001716614
+        "Да": 0.2134118527173996,
+        "Нет": 0.5801134705543518
     },
     "sample": {
         "messages": [
@@ -7725,8 +7725,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.9851431250572205,
-        "Нет": 0.014052311889827251
+        "Да": 0.8618310689926147,
+        "Нет": 0.11663615703582764
     },
     "sample": {
         "messages": [
@@ -7764,8 +7764,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0013247027527540922,
-        "Нет": 0.9984340071678162
+        "Да": 0.14564481377601624,
+        "Нет": 0.8381280899047852
     },
     "sample": {
         "messages": [
@@ -7803,8 +7803,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0010320350993424654,
-        "Нет": 0.9987781643867493
+        "Да": 0.34413546323776245,
+        "Нет": 0.6429296731948853
     },
     "sample": {
         "messages": [
@@ -7842,8 +7842,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.8922936916351318,
-        "Нет": 0.10656928271055222
+        "Да": 0.6388486623764038,
+        "Нет": 0.34195107221603394
     },
     "sample": {
         "messages": [
@@ -7881,8 +7881,8 @@
         "acc": false
     },
     "predict": {
-        "Да": 0.0005525772576220334,
-        "Нет": 0.9990831613540649
+        "Да": 0.13097181916236877,
+        "Нет": 0.8540434837341309
     },
     "sample": {
         "messages": [
@@ -7920,8 +7920,8 @@
         "acc": true
     },
     "predict": {
-        "Да": 0.00010887620010180399,
-        "Нет": 0.9997009038925171
+        "Да": 0.21951821446418762,
+        "Нет": 0.7661938071250916
     },
     "sample": {
         "messages": [

llmtf_eval/darumeru_RWSD_params.jsonl

@@ -1,7 +1,7 @@
 {
     "custom_generation_config": null,
     "model_params": {
-        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
         "generation_config": {
             "bos_token_id": 145109,
             "do_sample": true,
@@ -36,7 +36,7 @@
         },
         "load_in_8bit": false,
         "torch_dtype": "auto",
-        "use_flash_attention_2": true,
+        "attn_implementation": "flash_attention_2",
         "device_map": "cuda:0",
         "use_fast_tokenizer": true,
         "leading_space": false,
@@ -47,7 +47,7 @@
     "task_params": {
         "max_len": 4000,
         "few_shot_count": 0,
-        "batch_size": 2,
+        "batch_size": 16,
         "max_sample_per_dataset": 10000000000000,
         "method": "calculate_tokens_proba"
     }

llmtf_eval/darumeru_RWSD_total.jsonl

@@ -1,7 +1,7 @@
 {
     "task_name": "darumeru/RWSD",
     "results": {
-        "acc": 0.5098039215686274
+        "acc": 0.49019607843137253
     },
-    "leaderboard_result": 0.5098039215686274
+    "leaderboard_result": 0.49019607843137253
 }

llmtf_eval/darumeru_USE_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 64,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval/darumeru_USE_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/USE",
+    "results": {
+        "grade_norm": 0.06078431372549018
+    },
+    "leaderboard_result": 0.06078431372549018
+}

llmtf_eval/darumeru_cp_para_ru_params.jsonl

@@ -1,7 +1,7 @@
 {
     "custom_generation_config": null,
     "model_params": {
-        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
         "generation_config": {
             "bos_token_id": 145109,
             "do_sample": true,
@@ -36,7 +36,7 @@
         },
         "load_in_8bit": false,
         "torch_dtype": "auto",
-        "use_flash_attention_2": true,
+        "attn_implementation": "flash_attention_2",
         "device_map": "cuda:0",
         "use_fast_tokenizer": true,
         "leading_space": false,
@@ -47,7 +47,7 @@
     "task_params": {
         "max_len": 4000,
         "few_shot_count": 0,
-        "batch_size": 2,
+        "batch_size": 16,
         "max_sample_per_dataset": 10000000000000,
         "method": "generate"
     }

llmtf_eval/darumeru_cp_para_ru_total.jsonl

@@ -1,9 +1,10 @@
 {
     "task_name": "darumeru/cp_para_ru",
     "results": {
-        "symbol_per_token": 3.8978256135744003,
-        "len": 0.8764597602663033,
-        "lcs": 0.05
+        "tokens_per_word": 1.905314928817744,
+        "symbol_per_token": 3.913951487866651,
+        "len": 0.9904780330832407,
+        "lcs": 0.8
     },
-    "leaderboard_result": 0.05
+    "leaderboard_result": 0.8
 }

llmtf_eval/evaluation_log.txt

@@ -1,251 +1,272 @@
-INFO: 2024-11-18 14:18:48,851: llmtf.base.evaluator: Starting eval on ['darumeru/multiq']
-INFO: 2024-11-18 14:18:48,852: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:18:48,852: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:18:50,653: llmtf.base.evaluator: Starting eval on ['darumeru/parus']
-INFO: 2024-11-18 14:18:50,654: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:18:50,654: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:18:52,696: llmtf.base.darumeru/MultiQ: Loading Dataset: 3.84s
-INFO: 2024-11-18 14:18:52,936: llmtf.base.evaluator: Starting eval on ['darumeru/rcb']
-INFO: 2024-11-18 14:18:52,936: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:18:52,936: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:18:53,102: llmtf.base.darumeru/PARus: Loading Dataset: 2.45s
-INFO: 2024-11-18 14:18:54,811: llmtf.base.evaluator: Starting eval on ['darumeru/ruopenbookqa']
-INFO: 2024-11-18 14:18:54,811: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:18:54,811: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:18:55,708: llmtf.base.darumeru/RCB: Loading Dataset: 2.77s
-INFO: 2024-11-18 14:18:56,455: llmtf.base.darumeru/PARus: Processing Dataset: 3.35s
-INFO: 2024-11-18 14:18:56,457: llmtf.base.darumeru/PARus: Results for darumeru/PARus:
-INFO: 2024-11-18 14:18:56,470: llmtf.base.darumeru/PARus: {'acc': 0.24}
-INFO: 2024-11-18 14:18:56,471: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:18:56,474: llmtf.base.evaluator: 
-mean	darumeru/PARus
-0.240	0.240
-INFO: 2024-11-18 14:18:56,487: llmtf.base.evaluator: Starting eval on ['darumeru/ruworldtree']
-INFO: 2024-11-18 14:18:56,488: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:18:56,488: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:18:58,099: llmtf.base.darumeru/ruOpenBookQA: Loading Dataset: 3.29s
-INFO: 2024-11-18 14:18:58,743: llmtf.base.evaluator: Starting eval on ['darumeru/rwsd']
-INFO: 2024-11-18 14:18:58,744: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:18:58,744: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:18:58,925: llmtf.base.darumeru/ruWorldTree: Loading Dataset: 2.44s
-INFO: 2024-11-18 14:19:00,968: llmtf.base.evaluator: Starting eval on ['daru/treewayextractive']
-INFO: 2024-11-18 14:19:00,968: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:19:00,968: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:19:01,031: llmtf.base.darumeru/RCB: Processing Dataset: 5.32s
-INFO: 2024-11-18 14:19:01,033: llmtf.base.darumeru/RCB: Results for darumeru/RCB:
-INFO: 2024-11-18 14:19:01,040: llmtf.base.darumeru/RCB: {'acc': 0.4727272727272727, 'f1_macro': 0.39356669305497743}
-INFO: 2024-11-18 14:19:01,041: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:19:01,044: llmtf.base.evaluator: 
-mean	darumeru/PARus	darumeru/RCB
-0.337	0.240	0.433
-INFO: 2024-11-18 14:19:01,497: llmtf.base.darumeru/RWSD: Loading Dataset: 2.75s
-INFO: 2024-11-18 14:19:01,851: llmtf.base.darumeru/ruWorldTree: Processing Dataset: 2.92s
-INFO: 2024-11-18 14:19:01,852: llmtf.base.darumeru/ruWorldTree: Results for darumeru/ruWorldTree:
-INFO: 2024-11-18 14:19:01,859: llmtf.base.darumeru/ruWorldTree: {'acc': 0.7714285714285715, 'f1_macro': 0.7726851851851853}
-INFO: 2024-11-18 14:19:01,859: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:19:01,863: llmtf.base.evaluator: 
-mean	darumeru/PARus	darumeru/RCB	darumeru/ruWorldTree
-0.482	0.240	0.433	0.772
-INFO: 2024-11-18 14:19:02,889: llmtf.base.evaluator: Starting eval on ['daru/treewayabstractive']
-INFO: 2024-11-18 14:19:02,890: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:19:02,890: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:19:03,199: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/rummlu']
-INFO: 2024-11-18 14:19:03,199: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:19:03,199: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:19:06,629: llmtf.base.darumeru/RWSD: Processing Dataset: 5.13s
-INFO: 2024-11-18 14:19:06,631: llmtf.base.darumeru/RWSD: Results for darumeru/RWSD:
-INFO: 2024-11-18 14:19:06,635: llmtf.base.darumeru/RWSD: {'acc': 0.5098039215686274}
-INFO: 2024-11-18 14:19:06,636: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:19:06,641: llmtf.base.evaluator: 
-mean	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/ruWorldTree
-0.489	0.240	0.433	0.510	0.772
-INFO: 2024-11-18 14:19:06,885: llmtf.base.daru/treewayabstractive: Loading Dataset: 4.00s
-INFO: 2024-11-18 14:19:07,496: llmtf.base.evaluator: Starting eval on ['darumeru/cp_para_ru']
-INFO: 2024-11-18 14:19:07,497: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:19:07,497: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:19:10,509: llmtf.base.darumeru/cp_para_ru: Loading Dataset: 3.01s
-INFO: 2024-11-18 14:19:13,909: llmtf.base.daru/treewayextractive: Loading Dataset: 12.94s
-INFO: 2024-11-18 14:19:44,800: llmtf.base.darumeru/ruOpenBookQA: Processing Dataset: 46.70s
-INFO: 2024-11-18 14:19:44,801: llmtf.base.darumeru/ruOpenBookQA: Results for darumeru/ruOpenBookQA:
-INFO: 2024-11-18 14:19:44,814: llmtf.base.darumeru/ruOpenBookQA: {'acc': 0.615979381443299, 'f1_macro': 0.6154023944317246}
-INFO: 2024-11-18 14:19:44,821: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:19:44,826: llmtf.base.evaluator: 
-mean	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/ruOpenBookQA	darumeru/ruWorldTree
-0.514	0.240	0.433	0.510	0.616	0.772
-INFO: 2024-11-18 14:21:05,969: llmtf.base.nlpcoreteam/ruMMLU: Loading Dataset: 122.77s
-INFO: 2024-11-18 14:21:35,520: llmtf.base.daru/treewayextractive: Processing Dataset: 141.61s
-INFO: 2024-11-18 14:21:35,523: llmtf.base.daru/treewayextractive: Results for daru/treewayextractive:
-INFO: 2024-11-18 14:21:35,753: llmtf.base.daru/treewayextractive: {'r-prec': 0.3782488455988456}
-INFO: 2024-11-18 14:21:35,793: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:21:35,799: llmtf.base.evaluator: 
-mean	daru/treewayextractive	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/ruOpenBookQA	darumeru/ruWorldTree
-0.491	0.378	0.240	0.433	0.510	0.616	0.772
-INFO: 2024-11-18 14:24:06,030: llmtf.base.darumeru/cp_para_ru: Processing Dataset: 295.52s
-INFO: 2024-11-18 14:24:06,032: llmtf.base.darumeru/cp_para_ru: Results for darumeru/cp_para_ru:
-INFO: 2024-11-18 14:24:06,036: llmtf.base.darumeru/cp_para_ru: {'symbol_per_token': 3.8978256135744003, 'len': 0.8764597602663033, 'lcs': 0.05}
-INFO: 2024-11-18 14:24:06,036: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:24:06,041: llmtf.base.evaluator: 
-mean	daru/treewayextractive	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_para_ru	darumeru/ruOpenBookQA	darumeru/ruWorldTree
-0.428	0.378	0.240	0.433	0.510	0.050	0.616	0.772
-INFO: 2024-11-18 14:24:17,174: llmtf.base.daru/treewayabstractive: Processing Dataset: 310.29s
-INFO: 2024-11-18 14:24:17,190: llmtf.base.daru/treewayabstractive: Results for daru/treewayabstractive:
-INFO: 2024-11-18 14:24:17,208: llmtf.base.daru/treewayabstractive: {'rouge1': 0.31023763628891676, 'rouge2': 0.09443696323171702}
-INFO: 2024-11-18 14:24:17,210: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:24:17,215: llmtf.base.evaluator: 
-mean	daru/treewayabstractive	daru/treewayextractive	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_para_ru	darumeru/ruOpenBookQA	darumeru/ruWorldTree
-0.400	0.202	0.378	0.240	0.433	0.510	0.050	0.616	0.772
-INFO: 2024-11-18 14:26:06,991: llmtf.base.nlpcoreteam/ruMMLU: Processing Dataset: 301.02s
-INFO: 2024-11-18 14:26:06,993: llmtf.base.nlpcoreteam/ruMMLU: Results for nlpcoreteam/ruMMLU:
-INFO: 2024-11-18 14:26:07,037: llmtf.base.nlpcoreteam/ruMMLU:                                        metric
+INFO: 2024-11-26 20:17:25,790: llmtf.base.evaluator: Starting eval on ['darumeru/multiq', 'darumeru/parus', 'darumeru/rcb', 'darumeru/rwsd', 'darumeru/use']
+INFO: 2024-11-26 20:17:25,791: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:25,791: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:27,525: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/rummlu']
+INFO: 2024-11-26 20:17:27,525: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:27,525: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:29,517: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/enmmlu']
+INFO: 2024-11-26 20:17:29,517: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:29,517: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:30,060: llmtf.base.darumeru/MultiQ: Loading Dataset: 4.27s
+INFO: 2024-11-26 20:17:31,597: llmtf.base.evaluator: Starting eval on ['daru/treewayabstractive']
+INFO: 2024-11-26 20:17:31,597: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:31,597: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:33,345: llmtf.base.evaluator: Starting eval on ['darumeru/cp_para_ru']
+INFO: 2024-11-26 20:17:33,345: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:33,345: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:35,521: llmtf.base.evaluator: Starting eval on ['vikhrmodels/habr_qa_sbs', 'ruparam', 'shlepa/moviesmc', 'shlepa/musicmc', 'shlepa/lawmc', 'shlepa/booksmc']
+INFO: 2024-11-26 20:17:35,521: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:35,521: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:36,308: llmtf.base.darumeru/cp_para_ru: Loading Dataset: 2.96s
+INFO: 2024-11-26 20:17:36,742: llmtf.base.daru/treewayabstractive: Loading Dataset: 5.14s
+INFO: 2024-11-26 20:17:37,659: llmtf.base.evaluator: Starting eval on ['ruopinionne']
+INFO: 2024-11-26 20:17:37,659: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:37,660: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:37,984: llmtf.base.ruopinionne: Loading Dataset: 0.32s
+INFO: 2024-11-26 20:17:39,551: llmtf.base.evaluator: Starting eval on ['nerel']
+INFO: 2024-11-26 20:17:39,551: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:17:39,551: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:17:43,423: llmtf.base.NEREL: Loading Dataset: 3.87s
+INFO: 2024-11-26 20:17:48,557: llmtf.base.vikhrmodels/habr_qa_sbs: Loading Dataset: 13.04s
+INFO: 2024-11-26 20:18:44,135: llmtf.base.darumeru/cp_para_ru: Processing Dataset: 67.83s
+INFO: 2024-11-26 20:18:44,145: llmtf.base.darumeru/cp_para_ru: Results for darumeru/cp_para_ru:
+INFO: 2024-11-26 20:18:44,149: llmtf.base.darumeru/cp_para_ru: {'tokens_per_word': 1.905314928817744, 'symbol_per_token': 3.913951487866651, 'len': 0.9904780330832407, 'lcs': 0.8}
+INFO: 2024-11-26 20:18:44,150: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:18:44,152: llmtf.base.evaluator: 
+mean	darumeru/cp_para_ru
+0.800	0.800
+INFO: 2024-11-26 20:18:50,857: llmtf.base.NEREL: Processing Dataset: 67.43s
+INFO: 2024-11-26 20:18:50,860: llmtf.base.NEREL: Results for NEREL:
+INFO: 2024-11-26 20:18:50,864: llmtf.base.NEREL: {'tp': 2.0, 'fp': 27.0, 'fn': 519.0, 'micro-f1': 0.00727272727272595}
+INFO: 2024-11-26 20:18:50,865: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:18:50,869: llmtf.base.evaluator: 
+mean	NEREL	darumeru/cp_para_ru
+0.404	0.007	0.800
+INFO: 2024-11-26 20:18:57,627: llmtf.base.daru/treewayabstractive: Processing Dataset: 80.88s
+INFO: 2024-11-26 20:18:57,629: llmtf.base.daru/treewayabstractive: Results for daru/treewayabstractive:
+INFO: 2024-11-26 20:18:57,632: llmtf.base.daru/treewayabstractive: {'rouge1': 0.3138417117532064, 'rouge2': 0.10462617373556911}
+INFO: 2024-11-26 20:18:57,634: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:18:57,637: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/cp_para_ru
+0.339	0.007	0.209	0.800
+INFO: 2024-11-26 20:19:13,379: llmtf.base.darumeru/MultiQ: Processing Dataset: 103.32s
+INFO: 2024-11-26 20:19:13,384: llmtf.base.darumeru/MultiQ: Results for darumeru/MultiQ:
+INFO: 2024-11-26 20:19:13,404: llmtf.base.darumeru/MultiQ: {'f1': 0.3016876852043635, 'em': 0.21319311663479923}
+INFO: 2024-11-26 20:19:13,412: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:19:13,412: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:19:15,573: llmtf.base.darumeru/PARus: Loading Dataset: 2.16s
+INFO: 2024-11-26 20:19:19,861: llmtf.base.darumeru/PARus: Processing Dataset: 4.29s
+INFO: 2024-11-26 20:19:19,867: llmtf.base.darumeru/PARus: Results for darumeru/PARus:
+INFO: 2024-11-26 20:19:19,887: llmtf.base.darumeru/PARus: {'acc': 0.44}
+INFO: 2024-11-26 20:19:19,888: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:19:19,888: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:19:22,295: llmtf.base.darumeru/RCB: Loading Dataset: 2.40s
+INFO: 2024-11-26 20:19:23,604: llmtf.base.ruopinionne: Processing Dataset: 105.62s
+INFO: 2024-11-26 20:19:23,610: llmtf.base.ruopinionne: Results for ruopinionne:
+INFO: 2024-11-26 20:19:23,639: llmtf.base.ruopinionne: {'f1': 0.02701209922104298}
+INFO: 2024-11-26 20:19:23,640: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:19:23,656: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/cp_para_ru	ruopinionne
+0.290	0.007	0.209	0.257	0.440	0.800	0.027
+INFO: 2024-11-26 20:19:27,714: llmtf.base.darumeru/RCB: Processing Dataset: 5.42s
+INFO: 2024-11-26 20:19:27,715: llmtf.base.darumeru/RCB: Results for darumeru/RCB:
+INFO: 2024-11-26 20:19:27,722: llmtf.base.darumeru/RCB: {'acc': 0.4590909090909091, 'f1_macro': 0.36910715356478985}
+INFO: 2024-11-26 20:19:27,724: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:19:27,724: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:19:29,690: llmtf.base.darumeru/RWSD: Loading Dataset: 1.96s
+INFO: 2024-11-26 20:19:34,850: llmtf.base.darumeru/RWSD: Processing Dataset: 5.16s
+INFO: 2024-11-26 20:19:34,855: llmtf.base.darumeru/RWSD: Results for darumeru/RWSD:
+INFO: 2024-11-26 20:19:34,858: llmtf.base.darumeru/RWSD: {'acc': 0.49019607843137253}
+INFO: 2024-11-26 20:19:34,859: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:19:34,859: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:19:38,002: llmtf.base.darumeru/USE: Loading Dataset: 3.14s
+INFO: 2024-11-26 20:19:52,797: llmtf.base.nlpcoreteam/enMMLU: Loading Dataset: 143.28s
+INFO: 2024-11-26 20:19:55,208: llmtf.base.nlpcoreteam/ruMMLU: Loading Dataset: 147.68s
+INFO: 2024-11-26 20:20:33,495: llmtf.base.vikhrmodels/habr_qa_sbs: Processing Dataset: 164.94s
+INFO: 2024-11-26 20:20:33,496: llmtf.base.vikhrmodels/habr_qa_sbs: Results for vikhrmodels/habr_qa_sbs:
+INFO: 2024-11-26 20:20:33,533: llmtf.base.vikhrmodels/habr_qa_sbs: {'acc': 0.547, 'f1_macro': 0.5280856541414141}
+INFO: 2024-11-26 20:20:33,546: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:20:33,547: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:20:40,434: llmtf.base.ruparam: Loading Dataset: 6.89s
+INFO: 2024-11-26 20:21:07,899: llmtf.base.darumeru/USE: Processing Dataset: 89.90s
+INFO: 2024-11-26 20:21:07,900: llmtf.base.darumeru/USE: Results for darumeru/USE:
+INFO: 2024-11-26 20:21:07,942: llmtf.base.darumeru/USE: {'grade_norm': 0.06078431372549018}
+INFO: 2024-11-26 20:21:07,948: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:21:08,016: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	ruopinionne	vikhrmodels/habr_qa_sbs
+0.324	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.027	0.538
+INFO: 2024-11-26 20:25:43,772: llmtf.base.ruparam: Processing Dataset: 303.34s
+INFO: 2024-11-26 20:25:43,788: llmtf.base.ruparam: Results for ruparam:
+INFO: 2024-11-26 20:25:44,038: llmtf.base.ruparam: {'acc': 0.21363220494053065}
+INFO: 2024-11-26 20:25:44,053: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:25:44,053: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:25:47,874: llmtf.base.shlepa/movie_mc: Loading Dataset: 3.82s
+INFO: 2024-11-26 20:26:05,103: llmtf.base.shlepa/movie_mc: Processing Dataset: 17.22s
+INFO: 2024-11-26 20:26:05,119: llmtf.base.shlepa/movie_mc: Results for shlepa/movie_mc:
+INFO: 2024-11-26 20:26:05,122: llmtf.base.shlepa/movie_mc: {'acc': 0.22453703703703703}
+INFO: 2024-11-26 20:26:05,130: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:26:05,130: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:26:08,736: llmtf.base.shlepa/music_mc: Loading Dataset: 3.60s
+INFO: 2024-11-26 20:26:26,413: llmtf.base.shlepa/music_mc: Processing Dataset: 17.68s
+INFO: 2024-11-26 20:26:26,416: llmtf.base.shlepa/music_mc: Results for shlepa/music_mc:
+INFO: 2024-11-26 20:26:26,435: llmtf.base.shlepa/music_mc: {'acc': 0.24468085106382978}
+INFO: 2024-11-26 20:26:26,438: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:26:26,439: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:26:31,058: llmtf.base.shlepa/law_mc: Loading Dataset: 4.62s
+INFO: 2024-11-26 20:27:15,973: llmtf.base.shlepa/law_mc: Processing Dataset: 44.91s
+INFO: 2024-11-26 20:27:15,980: llmtf.base.shlepa/law_mc: Results for shlepa/law_mc:
+INFO: 2024-11-26 20:27:16,000: llmtf.base.shlepa/law_mc: {'acc': 0.537590113285273}
+INFO: 2024-11-26 20:27:16,006: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
+INFO: 2024-11-26 20:27:16,006: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
+INFO: 2024-11-26 20:27:19,668: llmtf.base.shlepa/books_mc: Loading Dataset: 3.66s
+INFO: 2024-11-26 20:27:39,200: llmtf.base.shlepa/books_mc: Processing Dataset: 19.53s
+INFO: 2024-11-26 20:27:39,204: llmtf.base.shlepa/books_mc: Results for shlepa/books_mc:
+INFO: 2024-11-26 20:27:39,209: llmtf.base.shlepa/books_mc: {'acc': 0.3112033195020747}
+INFO: 2024-11-26 20:27:39,212: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:27:39,231: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	ruopinionne	ruparam	shlepa/books_mc	shlepa/law_mc	shlepa/movie_mc	shlepa/music_mc	vikhrmodels/habr_qa_sbs
+0.318	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.027	0.214	0.311	0.538	0.225	0.245	0.538
+INFO: 2024-11-26 20:28:00,389: llmtf.base.nlpcoreteam/enMMLU: Processing Dataset: 487.59s
+INFO: 2024-11-26 20:28:00,396: llmtf.base.nlpcoreteam/enMMLU: Results for nlpcoreteam/enMMLU:
+INFO: 2024-11-26 20:28:00,445: llmtf.base.nlpcoreteam/enMMLU:                                        metric
 subject                                      
-abstract_algebra                     0.280000
-anatomy                              0.400000
-astronomy                            0.572368
-business_ethics                      0.460000
-clinical_knowledge                   0.494340
-college_biology                      0.375000
-college_chemistry                    0.290000
-college_computer_science             0.400000
-college_mathematics                  0.400000
-college_medicine                     0.491329
-college_physics                      0.362745
-computer_security                    0.500000
-conceptual_physics                   0.421277
-econometrics                         0.280702
-electrical_engineering               0.427586
-elementary_mathematics               0.391534
-formal_logic                         0.373016
-global_facts                         0.230000
-high_school_biology                  0.496774
-high_school_chemistry                0.458128
-high_school_computer_science         0.500000
-high_school_european_history         0.600000
-high_school_geography                0.535354
-high_school_government_and_politics  0.518135
-high_school_macroeconomics           0.471795
-high_school_mathematics              0.400000
-high_school_microeconomics           0.462185
-high_school_physics                  0.291391
-high_school_psychology               0.614679
-high_school_statistics               0.490741
-high_school_us_history               0.534314
-high_school_world_history            0.624473
-human_aging                          0.520179
-human_sexuality                      0.519084
-international_law                    0.694215
-jurisprudence                        0.537037
-logical_fallacies                    0.472393
-machine_learning                     0.258929
-management                           0.640777
-marketing                            0.700855
-medical_genetics                     0.480000
-miscellaneous                        0.533844
-moral_disputes                       0.488439
-moral_scenarios                      0.268156
-nutrition                            0.526144
-philosophy                           0.543408
-prehistory                           0.475309
-professional_accounting              0.347518
-professional_law                     0.345502
-professional_medicine                0.426471
-professional_psychology              0.411765
-public_relations                     0.427273
-security_studies                     0.542857
-sociology                            0.686567
-us_foreign_policy                    0.700000
-virology                             0.379518
-world_religions                      0.538012
-INFO: 2024-11-18 14:26:07,045: llmtf.base.nlpcoreteam/ruMMLU:                                    metric
-subject                                  
-STEM                             0.406471
-humanities                       0.499559
-other (business, health, misc.)  0.473641
-social sciences                  0.514200
-INFO: 2024-11-18 14:26:07,053: llmtf.base.nlpcoreteam/ruMMLU: {'acc': 0.47346768524963356}
-INFO: 2024-11-18 14:26:07,087: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:26:07,096: llmtf.base.evaluator: 
-mean	daru/treewayabstractive	daru/treewayextractive	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_para_ru	darumeru/ruOpenBookQA	darumeru/ruWorldTree	nlpcoreteam/ruMMLU
-0.408	0.202	0.378	0.240	0.433	0.510	0.050	0.616	0.772	0.473
-INFO: 2024-11-18 14:29:49,600: llmtf.base.darumeru/MultiQ: Processing Dataset: 656.90s
-INFO: 2024-11-18 14:29:49,603: llmtf.base.darumeru/MultiQ: Results for darumeru/MultiQ:
-INFO: 2024-11-18 14:29:49,608: llmtf.base.darumeru/MultiQ: {'f1': 0.20613243758223346, 'em': 0.11281070745697896}
-INFO: 2024-11-18 14:29:49,612: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:29:49,634: llmtf.base.evaluator: 
-mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_para_ru	darumeru/ruOpenBookQA	darumeru/ruWorldTree	nlpcoreteam/ruMMLU
-0.383	0.202	0.378	0.159	0.240	0.433	0.510	0.050	0.616	0.772	0.473
-INFO: 2024-11-18 14:29:55,578: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/enmmlu']
-INFO: 2024-11-18 14:29:55,579: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [145111]
-INFO: 2024-11-18 14:29:55,579: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['<|im_end|>']
-INFO: 2024-11-18 14:31:56,928: llmtf.base.nlpcoreteam/enMMLU: Loading Dataset: 121.35s
-INFO: 2024-11-18 14:36:48,348: llmtf.base.nlpcoreteam/enMMLU: Processing Dataset: 291.42s
-INFO: 2024-11-18 14:36:48,352: llmtf.base.nlpcoreteam/enMMLU: Results for nlpcoreteam/enMMLU:
-INFO: 2024-11-18 14:36:48,396: llmtf.base.nlpcoreteam/enMMLU:                                        metric
-subject                                      
-abstract_algebra                     0.350000
-anatomy                              0.562963
-astronomy                            0.664474
-business_ethics                      0.640000
-clinical_knowledge                   0.652830
-college_biology                      0.652778
-college_chemistry                    0.450000
-college_computer_science             0.490000
-college_mathematics                  0.310000
+abstract_algebra                     0.310000
+anatomy                              0.518519
+astronomy                            0.717105
+business_ethics                      0.630000
+clinical_knowledge                   0.667925
+college_biology                      0.680556
+college_chemistry                    0.400000
+college_computer_science             0.510000
+college_mathematics                  0.270000
 college_medicine                     0.618497
-college_physics                      0.500000
-computer_security                    0.710000
-conceptual_physics                   0.587234
-econometrics                         0.429825
-electrical_engineering               0.531034
-elementary_mathematics               0.460317
-formal_logic                         0.373016
-global_facts                         0.270000
-high_school_biology                  0.748387
-high_school_chemistry                0.522167
-high_school_computer_science         0.620000
-high_school_european_history         0.733333
-high_school_geography                0.747475
+college_physics                      0.558824
+computer_security                    0.750000
+conceptual_physics                   0.561702
+econometrics                         0.438596
+electrical_engineering               0.579310
+elementary_mathematics               0.455026
+formal_logic                         0.412698
+global_facts                         0.250000
+high_school_biology                  0.738710
+high_school_chemistry                0.541872
+high_school_computer_science         0.650000
+high_school_european_history         0.751515
+high_school_geography                0.767677
 high_school_government_and_politics  0.808290
-high_school_macroeconomics           0.658974
-high_school_mathematics              0.403704
-high_school_microeconomics           0.684874
-high_school_physics                  0.390728
-high_school_psychology               0.822018
-high_school_statistics               0.550926
-high_school_us_history               0.720588
-high_school_world_history            0.742616
-human_aging                          0.623318
-human_sexuality                      0.687023
-international_law                    0.710744
-jurisprudence                        0.759259
-logical_fallacies                    0.742331
-machine_learning                     0.419643
-management                           0.747573
-marketing                            0.824786
+high_school_macroeconomics           0.638462
+high_school_mathematics              0.433333
+high_school_microeconomics           0.676471
+high_school_physics                  0.377483
+high_school_psychology               0.814679
+high_school_statistics               0.527778
+high_school_us_history               0.715686
+high_school_world_history            0.767932
+human_aging                          0.641256
+human_sexuality                      0.679389
+international_law                    0.735537
+jurisprudence                        0.777778
+logical_fallacies                    0.760736
+machine_learning                     0.455357
+management                           0.757282
+marketing                            0.837607
 medical_genetics                     0.690000
-miscellaneous                        0.708812
-moral_disputes                       0.641618
-moral_scenarios                      0.252514
-nutrition                            0.653595
+miscellaneous                        0.713921
+moral_disputes                       0.650289
+moral_scenarios                      0.243575
+nutrition                            0.663399
 philosophy                           0.668810
-prehistory                           0.675926
-professional_accounting              0.510638
-professional_law                     0.397653
-professional_medicine                0.602941
-professional_psychology              0.589869
-public_relations                     0.609091
-security_studies                     0.673469
-sociology                            0.776119
-us_foreign_policy                    0.740000
+prehistory                           0.682099
+professional_accounting              0.489362
+professional_law                     0.418514
+professional_medicine                0.599265
+professional_psychology              0.588235
+public_relations                     0.600000
+security_studies                     0.689796
+sociology                            0.786070
+us_foreign_policy                    0.780000
 virology                             0.463855
-world_religions                      0.783626
-INFO: 2024-11-18 14:36:48,403: llmtf.base.nlpcoreteam/enMMLU:                                    metric
+world_religions                      0.789474
+INFO: 2024-11-26 20:28:00,453: llmtf.base.nlpcoreteam/enMMLU:                                    metric
+subject                                  
+STEM                             0.528725
+humanities                       0.644203
+other (business, health, misc.)  0.610063
+social sciences                  0.688972
+INFO: 2024-11-26 20:28:00,464: llmtf.base.nlpcoreteam/enMMLU: {'acc': 0.6179910195383496}
+INFO: 2024-11-26 20:28:00,501: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:28:00,524: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	nlpcoreteam/enMMLU	ruopinionne	ruparam	shlepa/books_mc	shlepa/law_mc	shlepa/movie_mc	shlepa/music_mc	vikhrmodels/habr_qa_sbs
+0.337	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.618	0.027	0.214	0.311	0.538	0.225	0.245	0.538
+INFO: 2024-11-26 20:28:16,900: llmtf.base.nlpcoreteam/ruMMLU: Processing Dataset: 501.69s
+INFO: 2024-11-26 20:28:16,902: llmtf.base.nlpcoreteam/ruMMLU: Results for nlpcoreteam/ruMMLU:
+INFO: 2024-11-26 20:28:16,950: llmtf.base.nlpcoreteam/ruMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.320000
+anatomy                              0.414815
+astronomy                            0.572368
+business_ethics                      0.450000
+clinical_knowledge                   0.505660
+college_biology                      0.375000
+college_chemistry                    0.310000
+college_computer_science             0.380000
+college_mathematics                  0.350000
+college_medicine                     0.508671
+college_physics                      0.431373
+computer_security                    0.530000
+conceptual_physics                   0.429787
+econometrics                         0.298246
+electrical_engineering               0.448276
+elementary_mathematics               0.417989
+formal_logic                         0.365079
+global_facts                         0.240000
+high_school_biology                  0.487097
+high_school_chemistry                0.443350
+high_school_computer_science         0.530000
+high_school_european_history         0.654545
+high_school_geography                0.525253
+high_school_government_and_politics  0.481865
+high_school_macroeconomics           0.430769
+high_school_mathematics              0.392593
+high_school_microeconomics           0.441176
+high_school_physics                  0.291391
+high_school_psychology               0.572477
+high_school_statistics               0.416667
+high_school_us_history               0.495098
+high_school_world_history            0.632911
+human_aging                          0.488789
+human_sexuality                      0.496183
+international_law                    0.677686
+jurisprudence                        0.574074
+logical_fallacies                    0.441718
+machine_learning                     0.321429
+management                           0.563107
+marketing                            0.722222
+medical_genetics                     0.470000
+miscellaneous                        0.536398
+moral_disputes                       0.517341
+moral_scenarios                      0.237989
+nutrition                            0.526144
+philosophy                           0.546624
+prehistory                           0.478395
+professional_accounting              0.365248
+professional_law                     0.331160
+professional_medicine                0.367647
+professional_psychology              0.415033
+public_relations                     0.454545
+security_studies                     0.595918
+sociology                            0.631841
+us_foreign_policy                    0.710000
+virology                             0.409639
+world_religions                      0.549708
+INFO: 2024-11-26 20:28:16,959: llmtf.base.nlpcoreteam/ruMMLU:                                    metric
 subject                                  
-STEM                             0.520077
-humanities                       0.630926
-other (business, health, misc.)  0.612129
-social sciences                  0.685586
-INFO: 2024-11-18 14:36:48,425: llmtf.base.nlpcoreteam/enMMLU: {'acc': 0.6121795307919802}
-INFO: 2024-11-18 14:36:48,459: llmtf.base.evaluator: Ended eval
-INFO: 2024-11-18 14:36:48,480: llmtf.base.evaluator: 
-mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_para_ru	darumeru/ruOpenBookQA	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU
-0.404	0.202	0.378	0.159	0.240	0.433	0.510	0.050	0.616	0.772	0.612	0.473
+STEM                             0.413740
+humanities                       0.500179
+other (business, health, misc.)  0.469167
+social sciences                  0.504442
+INFO: 2024-11-26 20:28:16,967: llmtf.base.nlpcoreteam/ruMMLU: {'acc': 0.47188210698069283}
+INFO: 2024-11-26 20:28:17,010: llmtf.base.evaluator: Ended eval
+INFO: 2024-11-26 20:28:17,020: llmtf.base.evaluator: 
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	ruopinionne	ruparam	shlepa/books_mc	shlepa/law_mc	shlepa/movie_mc	shlepa/music_mc	vikhrmodels/habr_qa_sbs
+0.345	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.618	0.472	0.027	0.214	0.311	0.538	0.225	0.245	0.538

llmtf_eval/evaluation_results.txt

@@ -1,2 +1,2 @@
-mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_para_ru	darumeru/ruOpenBookQA	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU
-0.404	0.202	0.378	0.159	0.240	0.433	0.510	0.050	0.616	0.772	0.612	0.473
+mean	NEREL	daru/treewayabstractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	llm_as_judge	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	ruopinionne	ruparam	shlepa/books_mc	shlepa/law_mc	shlepa/movie_mc	shlepa/music_mc	vikhrmodels/habr_qa_sbs
+0.349	0.007	0.209	0.257	0.440	0.414	0.490	0.061	0.800	0.420	0.618	0.472	0.027	0.214	0.311	0.538	0.225	0.245	0.538

llmtf_eval/llm_as_judge_params.jsonl

@@ -0,0 +1,22 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/workdir/models/Qwen2.5-72B-Instruct",
+        "api_base": "http://10.36.60.73:3040",
+        "generation_config": {
+            "max_new_tokens": 200,
+            "temperature": 0.0,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2"
+        },
+        "max_model_len": 32000
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 256,
+        "max_sample_per_dataset": 10000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/llm_as_judge_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "llm_as_judge",
+    "results": {
+        "score": 0.42
+    },
+    "leaderboard_result": 0.42
+}

llmtf_eval/nlpcoreteam_enMMLU.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc82cd8c8f64da35f0862509ddfd17217e9fb107f450d2e3e2807f69ff6bc80c
-size 37142060
+oid sha256:d39638678b135a0ac575407993d8d413c727aa2d94d2de0aca2520744a2819ea
+size 37122790

llmtf_eval/nlpcoreteam_enMMLU_params.jsonl

@@ -1,7 +1,7 @@
 {
     "custom_generation_config": null,
     "model_params": {
-        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
         "generation_config": {
             "bos_token_id": 145109,
             "do_sample": true,
@@ -36,7 +36,7 @@
         },
         "load_in_8bit": false,
         "torch_dtype": "auto",
-        "use_flash_attention_2": true,
+        "attn_implementation": "flash_attention_2",
         "device_map": "cuda:0",
         "use_fast_tokenizer": true,
         "leading_space": false,
@@ -47,7 +47,7 @@
     "task_params": {
         "max_len": 4000,
         "few_shot_count": 0,
-        "batch_size": 2,
+        "batch_size": 16,
         "max_sample_per_dataset": 10000000000000,
         "method": "calculate_tokens_proba"
     }

llmtf_eval/nlpcoreteam_enMMLU_total.jsonl

@@ -1,7 +1,7 @@
 {
     "task_name": "nlpcoreteam/enMMLU",
     "results": {
-        "acc": 0.6121795307919802
+        "acc": 0.6179910195383496
     },
-    "leaderboard_result": 0.6121795307919802
+    "leaderboard_result": 0.6179910195383496
 }

llmtf_eval/nlpcoreteam_ruMMLU.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c07ebf724c9c50407ac459026f3c3514cc89efab52531b4f6c15c9002046f0ec
-size 43069976
+oid sha256:c8a67a489efec02331bf36f2d61f632f8e982c340cfc1657e5ff3aa9ccc82849
+size 43039512

llmtf_eval/nlpcoreteam_ruMMLU_params.jsonl

@@ -1,7 +1,7 @@
 {
     "custom_generation_config": null,
     "model_params": {
-        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.5_nm_pv21/simpo2",
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
         "generation_config": {
             "bos_token_id": 145109,
             "do_sample": true,
@@ -36,7 +36,7 @@
         },
         "load_in_8bit": false,
         "torch_dtype": "auto",
-        "use_flash_attention_2": true,
+        "attn_implementation": "flash_attention_2",
         "device_map": "cuda:0",
         "use_fast_tokenizer": true,
         "leading_space": false,
@@ -47,7 +47,7 @@
     "task_params": {
         "max_len": 4000,
         "few_shot_count": 0,
-        "batch_size": 2,
+        "batch_size": 16,
         "max_sample_per_dataset": 10000000000000,
         "method": "calculate_tokens_proba"
     }

llmtf_eval/nlpcoreteam_ruMMLU_total.jsonl

@@ -1,7 +1,7 @@
 {
     "task_name": "nlpcoreteam/ruMMLU",
     "results": {
-        "acc": 0.47346768524963356
+        "acc": 0.47188210698069283
     },
-    "leaderboard_result": 0.47346768524963356
+    "leaderboard_result": 0.47188210698069283
 }

llmtf_eval/ruopinionne_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 256,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 200,
+        "method": "generate"
+    }
+}

llmtf_eval/ruopinionne_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "ruopinionne",
+    "results": {
+        "f1": 0.02701209922104298
+    },
+    "leaderboard_result": 0.02701209922104298
+}

llmtf_eval/ruparam.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d45cd9b4dd24149a1964646b1c3f49d2eae1fa1f73f313a0bb433aac98a22973
+size 12306566

llmtf_eval/ruparam_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/ruparam_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "ruparam",
+    "results": {
+        "acc": 0.21363220494053065
+    },
+    "leaderboard_result": 0.21363220494053065
+}

llmtf_eval/shlepa_books_mc_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/shlepa_books_mc_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "shlepa/books_mc",
+    "results": {
+        "acc": 0.3112033195020747
+    },
+    "leaderboard_result": 0.3112033195020747
+}

llmtf_eval/shlepa_law_mc_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/ruadapt_qwen2.5_1.5B_ext_u48_part1-2_lr5e4_bs128_reinit_peft_mlp_32_128_lm_head_mts_1e4_as1.75_v1/simpo2",
+        "generation_config": {
+            "bos_token_id": 145109,
+            "do_sample": true,
+            "eos_token_id": [
+                145111
+            ],
+            "max_length": 32768,
+            "max_new_tokens": 1,
+            "pad_token_id": 145109,
+            "stop_strings": [
+                "<|im_end|>"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.45.2",
+            "trust_remote_code": false
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "user_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template": "<|im_start|>{role}\n{content}<|im_end|>\n",
+            "bot_message_template_incomplete": "<|im_start|>{role}\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "",
+            "suffix": "<|im_start|>assistant\n",
+            "add_special_tokens": false,
+            "eos_token": "<|im_end|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "attn_implementation": "flash_attention_2",
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": false,
+        "max_model_len": 32768
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 16,
+        "max_sample_per_dataset": 5000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval/shlepa_law_mc_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "shlepa/law_mc",
+    "results": {
+        "acc": 0.537590113285273
+    },
+    "leaderboard_result": 0.537590113285273
+}