Hugging Face's logo

ChrisMcCormick
/
SubspaceDecoder_mla0-0-0

Model card Files
xet
Community
ChrisMcCormick commited on
Commit
4a91cbd
·
verified ·
1 Parent(s): 29e64ff

Upload folder using huggingface_hub

Browse files
Files changed (25) hide show
  1. checkpoint-2700/config.json +43 -0
  2. checkpoint-2700/merges.txt +0 -0
  3. checkpoint-2700/optimizer.pt +3 -0
  4. checkpoint-2700/pytorch_model.bin +3 -0
  5. checkpoint-2700/rng_state.pth +3 -0
  6. checkpoint-2700/scheduler.pt +3 -0
  7. checkpoint-2700/special_tokens_map.json +6 -0
  8. checkpoint-2700/tokenizer.json +0 -0
  9. checkpoint-2700/tokenizer_config.json +21 -0
  10. checkpoint-2700/trainer_state.json +1060 -0
  11. checkpoint-2700/training_args.bin +3 -0
  12. checkpoint-2700/vocab.json +0 -0
  13. checkpoint-3000/config.json +43 -0
  14. checkpoint-3000/merges.txt +0 -0
  15. checkpoint-3000/optimizer.pt +3 -0
  16. checkpoint-3000/pytorch_model.bin +3 -0
  17. checkpoint-3000/rng_state.pth +3 -0
  18. checkpoint-3000/scheduler.pt +3 -0
  19. checkpoint-3000/special_tokens_map.json +6 -0
  20. checkpoint-3000/tokenizer.json +0 -0
  21. checkpoint-3000/tokenizer_config.json +21 -0
  22. checkpoint-3000/trainer_state.json +1174 -0
  23. checkpoint-3000/training_args.bin +3 -0
  24. checkpoint-3000/vocab.json +0 -0
  25. full_config.json +73 -0
checkpoint-2700/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "SharedSpaceDecoderForCausalLM"
4
+ ],
5
+ "attention_backend": "flash_attention_2",
6
+ "attention_bias": false,
7
+ "attention_dropout_prob": 0.1,
8
+ "bos_token_id": 50256,
9
+ "classifier_dropout": null,
10
+ "dtype": "float32",
11
+ "eos_token_id": 50256,
12
+ "ffn_decompose": false,
13
+ "ffn_rank": null,
14
+ "hidden_dropout_prob": 0.1,
15
+ "hidden_size": 768,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 2048,
18
+ "kv_shared_dim": null,
19
+ "layer_norm_eps": 1e-12,
20
+ "max_position_embeddings": 1024,
21
+ "model_type": "shared_subspace_decoder",
22
+ "nope_dims": 32,
23
+ "norm_type": "rmsnorm",
24
+ "num_attention_heads": 12,
25
+ "num_dense_layers": 0,
26
+ "num_hidden_layers": 12,
27
+ "o_shared_dim": null,
28
+ "pad_token_id": 50256,
29
+ "q_shared_dim": null,
30
+ "qk_private_dim": 64,
31
+ "rms_norm_eps": 1e-06,
32
+ "rope_dims": 32,
33
+ "rope_scaling": {
34
+ "factor": 2.0,
35
+ "type": "linear"
36
+ },
37
+ "rope_theta": 10000.0,
38
+ "transformers_version": "4.56.0",
39
+ "vo_private_dim": 64,
40
+ "vocab_rank": null,
41
+ "vocab_size": 50257,
42
+ "vocab_subspace": false
43
+ }
checkpoint-2700/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-2700/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f7a1430fc97b046dd29e562728ef516f253dba0862a4e8159c3b0f62449c3ba
3
+ size 988989899
checkpoint-2700/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb5ec96e21b65268860c6e701fc4420d405a47a2a6ccf3b86e30f1c05dcca018
3
+ size 494483579
checkpoint-2700/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c35fc70182e7ca9c2fecd3307516287efe33bd69963aae42900e793d8582f9b
3
+ size 14645
checkpoint-2700/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c42329fe9ce60834aecb564501f655ead9473db295ff06d83d914a3e71fedd4
3
+ size 1465
checkpoint-2700/special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "unk_token": "<|endoftext|>"
6
+ }
checkpoint-2700/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-2700/tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ }
12
+ },
13
+ "bos_token": "<|endoftext|>",
14
+ "clean_up_tokenization_spaces": false,
15
+ "eos_token": "<|endoftext|>",
16
+ "extra_special_tokens": {},
17
+ "model_max_length": 1024,
18
+ "pad_token": "<|endoftext|>",
19
+ "tokenizer_class": "GPT2Tokenizer",
20
+ "unk_token": "<|endoftext|>"
21
+ }
checkpoint-2700/trainer_state.json ADDED
@@ -0,0 +1,1060 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 2700,
3
+ "best_metric": 3.600645640400387,
4
+ "best_model_checkpoint": "checkpoints/gpt-2_seq1024_mla0-0-0/checkpoint-2700",
5
+ "epoch": 0.8138045362067666,
6
+ "eval_steps": 300,
7
+ "global_step": 2700,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.006028181749679753,
14
+ "grad_norm": 2.0441031455993652,
15
+ "learning_rate": 3.166666666666667e-05,
16
+ "loss": 10.3921,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.012056363499359506,
21
+ "grad_norm": 1.7166392803192139,
22
+ "learning_rate": 6.500000000000001e-05,
23
+ "loss": 9.241,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.01808454524903926,
28
+ "grad_norm": 0.7614754438400269,
29
+ "learning_rate": 9.833333333333333e-05,
30
+ "loss": 8.2219,
31
+ "step": 60
32
+ },
33
+ {
34
+ "epoch": 0.02411272699871901,
35
+ "grad_norm": 0.46116286516189575,
36
+ "learning_rate": 0.00013166666666666665,
37
+ "loss": 7.5368,
38
+ "step": 80
39
+ },
40
+ {
41
+ "epoch": 0.030140908748398764,
42
+ "grad_norm": 1.6069883108139038,
43
+ "learning_rate": 0.000165,
44
+ "loss": 7.21,
45
+ "step": 100
46
+ },
47
+ {
48
+ "epoch": 0.03616909049807852,
49
+ "grad_norm": 1.2077178955078125,
50
+ "learning_rate": 0.00019833333333333335,
51
+ "loss": 6.8976,
52
+ "step": 120
53
+ },
54
+ {
55
+ "epoch": 0.04219727224775827,
56
+ "grad_norm": 0.642643928527832,
57
+ "learning_rate": 0.00023166666666666667,
58
+ "loss": 6.641,
59
+ "step": 140
60
+ },
61
+ {
62
+ "epoch": 0.04822545399743802,
63
+ "grad_norm": 0.7632517218589783,
64
+ "learning_rate": 0.00026500000000000004,
65
+ "loss": 6.4194,
66
+ "step": 160
67
+ },
68
+ {
69
+ "epoch": 0.05425363574711778,
70
+ "grad_norm": 1.2542935609817505,
71
+ "learning_rate": 0.00029833333333333334,
72
+ "loss": 6.2428,
73
+ "step": 180
74
+ },
75
+ {
76
+ "epoch": 0.06028181749679753,
77
+ "grad_norm": 1.1738799810409546,
78
+ "learning_rate": 0.0003316666666666667,
79
+ "loss": 6.1026,
80
+ "step": 200
81
+ },
82
+ {
83
+ "epoch": 0.06630999924647728,
84
+ "grad_norm": 0.46843603253364563,
85
+ "learning_rate": 0.000365,
86
+ "loss": 5.9586,
87
+ "step": 220
88
+ },
89
+ {
90
+ "epoch": 0.07233818099615703,
91
+ "grad_norm": 0.3949953019618988,
92
+ "learning_rate": 0.00039833333333333333,
93
+ "loss": 5.8304,
94
+ "step": 240
95
+ },
96
+ {
97
+ "epoch": 0.07836636274583679,
98
+ "grad_norm": 1.0866719484329224,
99
+ "learning_rate": 0.0004316666666666667,
100
+ "loss": 5.7185,
101
+ "step": 260
102
+ },
103
+ {
104
+ "epoch": 0.08439454449551655,
105
+ "grad_norm": 0.34839189052581787,
106
+ "learning_rate": 0.000465,
107
+ "loss": 5.6037,
108
+ "step": 280
109
+ },
110
+ {
111
+ "epoch": 0.09042272624519629,
112
+ "grad_norm": 0.5791395902633667,
113
+ "learning_rate": 0.0004983333333333334,
114
+ "loss": 5.5003,
115
+ "step": 300
116
+ },
117
+ {
118
+ "epoch": 0.09042272624519629,
119
+ "eval_loss": 5.428642945231348,
120
+ "eval_perplexity": 227.839844387096,
121
+ "eval_runtime": 113.1707,
122
+ "eval_samples_per_second": 14.659,
123
+ "eval_steps_per_second": 0.23,
124
+ "step": 300
125
+ },
126
+ {
127
+ "epoch": 0.09645090799487605,
128
+ "grad_norm": 0.532134473323822,
129
+ "learning_rate": 0.0004964814814814814,
130
+ "loss": 5.3805,
131
+ "step": 320
132
+ },
133
+ {
134
+ "epoch": 0.1024790897445558,
135
+ "grad_norm": 0.5876124501228333,
136
+ "learning_rate": 0.0004927777777777777,
137
+ "loss": 5.2775,
138
+ "step": 340
139
+ },
140
+ {
141
+ "epoch": 0.10850727149423556,
142
+ "grad_norm": 0.4140375256538391,
143
+ "learning_rate": 0.0004890740740740741,
144
+ "loss": 5.193,
145
+ "step": 360
146
+ },
147
+ {
148
+ "epoch": 0.1145354532439153,
149
+ "grad_norm": 0.5529780387878418,
150
+ "learning_rate": 0.0004853703703703704,
151
+ "loss": 5.0912,
152
+ "step": 380
153
+ },
154
+ {
155
+ "epoch": 0.12056363499359506,
156
+ "grad_norm": 0.3331276774406433,
157
+ "learning_rate": 0.0004816666666666667,
158
+ "loss": 5.0266,
159
+ "step": 400
160
+ },
161
+ {
162
+ "epoch": 0.1265918167432748,
163
+ "grad_norm": 0.6916081309318542,
164
+ "learning_rate": 0.00047796296296296297,
165
+ "loss": 4.9375,
166
+ "step": 420
167
+ },
168
+ {
169
+ "epoch": 0.13261999849295456,
170
+ "grad_norm": 0.31799590587615967,
171
+ "learning_rate": 0.0004742592592592593,
172
+ "loss": 4.9026,
173
+ "step": 440
174
+ },
175
+ {
176
+ "epoch": 0.1386481802426343,
177
+ "grad_norm": 0.5735257267951965,
178
+ "learning_rate": 0.00047055555555555555,
179
+ "loss": 4.8075,
180
+ "step": 460
181
+ },
182
+ {
183
+ "epoch": 0.14467636199231407,
184
+ "grad_norm": 0.5078163146972656,
185
+ "learning_rate": 0.00046685185185185187,
186
+ "loss": 4.759,
187
+ "step": 480
188
+ },
189
+ {
190
+ "epoch": 0.15070454374199382,
191
+ "grad_norm": 0.3884360194206238,
192
+ "learning_rate": 0.00046314814814814813,
193
+ "loss": 4.6706,
194
+ "step": 500
195
+ },
196
+ {
197
+ "epoch": 0.15673272549167358,
198
+ "grad_norm": 0.5591968297958374,
199
+ "learning_rate": 0.00045944444444444445,
200
+ "loss": 4.627,
201
+ "step": 520
202
+ },
203
+ {
204
+ "epoch": 0.16276090724135334,
205
+ "grad_norm": 0.29390859603881836,
206
+ "learning_rate": 0.0004557407407407407,
207
+ "loss": 4.5691,
208
+ "step": 540
209
+ },
210
+ {
211
+ "epoch": 0.1687890889910331,
212
+ "grad_norm": 0.3524036407470703,
213
+ "learning_rate": 0.00045203703703703703,
214
+ "loss": 4.5066,
215
+ "step": 560
216
+ },
217
+ {
218
+ "epoch": 0.17481727074071282,
219
+ "grad_norm": 0.4607946574687958,
220
+ "learning_rate": 0.0004483333333333333,
221
+ "loss": 4.4661,
222
+ "step": 580
223
+ },
224
+ {
225
+ "epoch": 0.18084545249039258,
226
+ "grad_norm": 0.2523379623889923,
227
+ "learning_rate": 0.00044462962962962967,
228
+ "loss": 4.4331,
229
+ "step": 600
230
+ },
231
+ {
232
+ "epoch": 0.18084545249039258,
233
+ "eval_loss": 4.357323692651888,
234
+ "eval_perplexity": 78.04797429242741,
235
+ "eval_runtime": 112.5783,
236
+ "eval_samples_per_second": 14.736,
237
+ "eval_steps_per_second": 0.231,
238
+ "step": 600
239
+ },
240
+ {
241
+ "epoch": 0.18687363424007233,
242
+ "grad_norm": 0.3953287601470947,
243
+ "learning_rate": 0.00044092592592592594,
244
+ "loss": 4.3877,
245
+ "step": 620
246
+ },
247
+ {
248
+ "epoch": 0.1929018159897521,
249
+ "grad_norm": 0.38012024760246277,
250
+ "learning_rate": 0.00043722222222222225,
251
+ "loss": 4.3619,
252
+ "step": 640
253
+ },
254
+ {
255
+ "epoch": 0.19892999773943185,
256
+ "grad_norm": 0.3135126233100891,
257
+ "learning_rate": 0.0004335185185185185,
258
+ "loss": 4.3442,
259
+ "step": 660
260
+ },
261
+ {
262
+ "epoch": 0.2049581794891116,
263
+ "grad_norm": 0.49422895908355713,
264
+ "learning_rate": 0.00042981481481481484,
265
+ "loss": 4.3045,
266
+ "step": 680
267
+ },
268
+ {
269
+ "epoch": 0.21098636123879136,
270
+ "grad_norm": 0.27019643783569336,
271
+ "learning_rate": 0.0004261111111111111,
272
+ "loss": 4.2808,
273
+ "step": 700
274
+ },
275
+ {
276
+ "epoch": 0.21701454298847112,
277
+ "grad_norm": 0.3935423791408539,
278
+ "learning_rate": 0.0004224074074074074,
279
+ "loss": 4.2467,
280
+ "step": 720
281
+ },
282
+ {
283
+ "epoch": 0.22304272473815084,
284
+ "grad_norm": 0.28830042481422424,
285
+ "learning_rate": 0.0004187037037037037,
286
+ "loss": 4.2274,
287
+ "step": 740
288
+ },
289
+ {
290
+ "epoch": 0.2290709064878306,
291
+ "grad_norm": 0.2877708673477173,
292
+ "learning_rate": 0.000415,
293
+ "loss": 4.2075,
294
+ "step": 760
295
+ },
296
+ {
297
+ "epoch": 0.23509908823751036,
298
+ "grad_norm": 0.3449930250644684,
299
+ "learning_rate": 0.00041129629629629627,
300
+ "loss": 4.1909,
301
+ "step": 780
302
+ },
303
+ {
304
+ "epoch": 0.2411272699871901,
305
+ "grad_norm": 0.326265424489975,
306
+ "learning_rate": 0.00040759259259259264,
307
+ "loss": 4.1684,
308
+ "step": 800
309
+ },
310
+ {
311
+ "epoch": 0.24715545173686987,
312
+ "grad_norm": 0.3821180760860443,
313
+ "learning_rate": 0.0004038888888888889,
314
+ "loss": 4.1513,
315
+ "step": 820
316
+ },
317
+ {
318
+ "epoch": 0.2531836334865496,
319
+ "grad_norm": 0.2768200635910034,
320
+ "learning_rate": 0.0004001851851851852,
321
+ "loss": 4.1317,
322
+ "step": 840
323
+ },
324
+ {
325
+ "epoch": 0.2592118152362294,
326
+ "grad_norm": 0.29724735021591187,
327
+ "learning_rate": 0.0003964814814814815,
328
+ "loss": 4.1171,
329
+ "step": 860
330
+ },
331
+ {
332
+ "epoch": 0.2652399969859091,
333
+ "grad_norm": 0.2814749479293823,
334
+ "learning_rate": 0.0003927777777777778,
335
+ "loss": 4.1049,
336
+ "step": 880
337
+ },
338
+ {
339
+ "epoch": 0.2712681787355889,
340
+ "grad_norm": 0.26310989260673523,
341
+ "learning_rate": 0.00038907407407407407,
342
+ "loss": 4.0858,
343
+ "step": 900
344
+ },
345
+ {
346
+ "epoch": 0.2712681787355889,
347
+ "eval_loss": 4.030442206068737,
348
+ "eval_perplexity": 56.285795665128816,
349
+ "eval_runtime": 112.6912,
350
+ "eval_samples_per_second": 14.722,
351
+ "eval_steps_per_second": 0.231,
352
+ "step": 900
353
+ },
354
+ {
355
+ "epoch": 0.2772963604852686,
356
+ "grad_norm": 0.3063352108001709,
357
+ "learning_rate": 0.0003853703703703704,
358
+ "loss": 4.0813,
359
+ "step": 920
360
+ },
361
+ {
362
+ "epoch": 0.2833245422349484,
363
+ "grad_norm": 0.4063352346420288,
364
+ "learning_rate": 0.00038166666666666666,
365
+ "loss": 4.0647,
366
+ "step": 940
367
+ },
368
+ {
369
+ "epoch": 0.28935272398462814,
370
+ "grad_norm": 0.2601182758808136,
371
+ "learning_rate": 0.000377962962962963,
372
+ "loss": 4.0447,
373
+ "step": 960
374
+ },
375
+ {
376
+ "epoch": 0.29538090573430786,
377
+ "grad_norm": 0.3153649568557739,
378
+ "learning_rate": 0.00037425925925925924,
379
+ "loss": 4.0417,
380
+ "step": 980
381
+ },
382
+ {
383
+ "epoch": 0.30140908748398765,
384
+ "grad_norm": 0.26004716753959656,
385
+ "learning_rate": 0.0003705555555555556,
386
+ "loss": 4.0279,
387
+ "step": 1000
388
+ },
389
+ {
390
+ "epoch": 0.3074372692336674,
391
+ "grad_norm": 0.43524181842803955,
392
+ "learning_rate": 0.0003668518518518519,
393
+ "loss": 4.0152,
394
+ "step": 1020
395
+ },
396
+ {
397
+ "epoch": 0.31346545098334716,
398
+ "grad_norm": 0.2601913511753082,
399
+ "learning_rate": 0.0003631481481481482,
400
+ "loss": 4.0026,
401
+ "step": 1040
402
+ },
403
+ {
404
+ "epoch": 0.3194936327330269,
405
+ "grad_norm": 0.25712424516677856,
406
+ "learning_rate": 0.00035944444444444446,
407
+ "loss": 3.9933,
408
+ "step": 1060
409
+ },
410
+ {
411
+ "epoch": 0.3255218144827067,
412
+ "grad_norm": 0.3313097357749939,
413
+ "learning_rate": 0.0003557407407407408,
414
+ "loss": 3.9842,
415
+ "step": 1080
416
+ },
417
+ {
418
+ "epoch": 0.3315499962323864,
419
+ "grad_norm": 0.238124281167984,
420
+ "learning_rate": 0.00035203703703703704,
421
+ "loss": 3.9745,
422
+ "step": 1100
423
+ },
424
+ {
425
+ "epoch": 0.3375781779820662,
426
+ "grad_norm": 0.28039032220840454,
427
+ "learning_rate": 0.00034833333333333336,
428
+ "loss": 3.9588,
429
+ "step": 1120
430
+ },
431
+ {
432
+ "epoch": 0.3436063597317459,
433
+ "grad_norm": 0.3661600649356842,
434
+ "learning_rate": 0.0003446296296296296,
435
+ "loss": 3.9532,
436
+ "step": 1140
437
+ },
438
+ {
439
+ "epoch": 0.34963454148142564,
440
+ "grad_norm": 0.22854666411876678,
441
+ "learning_rate": 0.0003409259259259259,
442
+ "loss": 3.9392,
443
+ "step": 1160
444
+ },
445
+ {
446
+ "epoch": 0.3556627232311054,
447
+ "grad_norm": 0.2930833697319031,
448
+ "learning_rate": 0.0003372222222222222,
449
+ "loss": 3.9312,
450
+ "step": 1180
451
+ },
452
+ {
453
+ "epoch": 0.36169090498078516,
454
+ "grad_norm": 0.25500425696372986,
455
+ "learning_rate": 0.0003335185185185185,
456
+ "loss": 3.9295,
457
+ "step": 1200
458
+ },
459
+ {
460
+ "epoch": 0.36169090498078516,
461
+ "eval_loss": 3.874079793811062,
462
+ "eval_perplexity": 48.13838066773452,
463
+ "eval_runtime": 112.5895,
464
+ "eval_samples_per_second": 14.735,
465
+ "eval_steps_per_second": 0.231,
466
+ "step": 1200
467
+ },
468
+ {
469
+ "epoch": 0.36771908673046494,
470
+ "grad_norm": 0.27924397587776184,
471
+ "learning_rate": 0.00032981481481481485,
472
+ "loss": 3.9157,
473
+ "step": 1220
474
+ },
475
+ {
476
+ "epoch": 0.37374726848014467,
477
+ "grad_norm": 0.2649654448032379,
478
+ "learning_rate": 0.0003261111111111111,
479
+ "loss": 3.9136,
480
+ "step": 1240
481
+ },
482
+ {
483
+ "epoch": 0.37977545022982445,
484
+ "grad_norm": 0.27920904755592346,
485
+ "learning_rate": 0.00032240740740740743,
486
+ "loss": 3.899,
487
+ "step": 1260
488
+ },
489
+ {
490
+ "epoch": 0.3858036319795042,
491
+ "grad_norm": 0.238219752907753,
492
+ "learning_rate": 0.0003187037037037037,
493
+ "loss": 3.8965,
494
+ "step": 1280
495
+ },
496
+ {
497
+ "epoch": 0.3918318137291839,
498
+ "grad_norm": 0.2615567743778229,
499
+ "learning_rate": 0.000315,
500
+ "loss": 3.8877,
501
+ "step": 1300
502
+ },
503
+ {
504
+ "epoch": 0.3978599954788637,
505
+ "grad_norm": 0.266513466835022,
506
+ "learning_rate": 0.0003112962962962963,
507
+ "loss": 3.8806,
508
+ "step": 1320
509
+ },
510
+ {
511
+ "epoch": 0.4038881772285434,
512
+ "grad_norm": 0.3212108314037323,
513
+ "learning_rate": 0.0003075925925925926,
514
+ "loss": 3.8774,
515
+ "step": 1340
516
+ },
517
+ {
518
+ "epoch": 0.4099163589782232,
519
+ "grad_norm": 0.231920063495636,
520
+ "learning_rate": 0.00030388888888888886,
521
+ "loss": 3.8707,
522
+ "step": 1360
523
+ },
524
+ {
525
+ "epoch": 0.41594454072790293,
526
+ "grad_norm": 0.27022039890289307,
527
+ "learning_rate": 0.0003001851851851852,
528
+ "loss": 3.863,
529
+ "step": 1380
530
+ },
531
+ {
532
+ "epoch": 0.4219727224775827,
533
+ "grad_norm": 0.243785560131073,
534
+ "learning_rate": 0.00029648148148148144,
535
+ "loss": 3.8562,
536
+ "step": 1400
537
+ },
538
+ {
539
+ "epoch": 0.42800090422726245,
540
+ "grad_norm": 0.2512820065021515,
541
+ "learning_rate": 0.0002927777777777778,
542
+ "loss": 3.8541,
543
+ "step": 1420
544
+ },
545
+ {
546
+ "epoch": 0.43402908597694223,
547
+ "grad_norm": 0.25888124108314514,
548
+ "learning_rate": 0.0002890740740740741,
549
+ "loss": 3.8421,
550
+ "step": 1440
551
+ },
552
+ {
553
+ "epoch": 0.44005726772662196,
554
+ "grad_norm": 0.29554685950279236,
555
+ "learning_rate": 0.0002853703703703704,
556
+ "loss": 3.8468,
557
+ "step": 1460
558
+ },
559
+ {
560
+ "epoch": 0.4460854494763017,
561
+ "grad_norm": 0.2608514130115509,
562
+ "learning_rate": 0.00028166666666666666,
563
+ "loss": 3.837,
564
+ "step": 1480
565
+ },
566
+ {
567
+ "epoch": 0.45211363122598147,
568
+ "grad_norm": 0.2781771123409271,
569
+ "learning_rate": 0.000277962962962963,
570
+ "loss": 3.8327,
571
+ "step": 1500
572
+ },
573
+ {
574
+ "epoch": 0.45211363122598147,
575
+ "eval_loss": 3.776173532413324,
576
+ "eval_perplexity": 43.6487014373908,
577
+ "eval_runtime": 112.7037,
578
+ "eval_samples_per_second": 14.72,
579
+ "eval_steps_per_second": 0.231,
580
+ "step": 1500
581
+ },
582
+ {
583
+ "epoch": 0.4581418129756612,
584
+ "grad_norm": 0.27767959237098694,
585
+ "learning_rate": 0.00027425925925925925,
586
+ "loss": 3.8264,
587
+ "step": 1520
588
+ },
589
+ {
590
+ "epoch": 0.464169994725341,
591
+ "grad_norm": 0.24143943190574646,
592
+ "learning_rate": 0.00027055555555555557,
593
+ "loss": 3.8224,
594
+ "step": 1540
595
+ },
596
+ {
597
+ "epoch": 0.4701981764750207,
598
+ "grad_norm": 0.23281621932983398,
599
+ "learning_rate": 0.00026685185185185183,
600
+ "loss": 3.8114,
601
+ "step": 1560
602
+ },
603
+ {
604
+ "epoch": 0.4762263582247005,
605
+ "grad_norm": 0.23552870750427246,
606
+ "learning_rate": 0.00026314814814814815,
607
+ "loss": 3.8086,
608
+ "step": 1580
609
+ },
610
+ {
611
+ "epoch": 0.4822545399743802,
612
+ "grad_norm": 0.25271207094192505,
613
+ "learning_rate": 0.0002594444444444444,
614
+ "loss": 3.7954,
615
+ "step": 1600
616
+ },
617
+ {
618
+ "epoch": 0.48828272172405995,
619
+ "grad_norm": 0.2609969675540924,
620
+ "learning_rate": 0.0002557407407407408,
621
+ "loss": 3.8052,
622
+ "step": 1620
623
+ },
624
+ {
625
+ "epoch": 0.49431090347373974,
626
+ "grad_norm": 0.290229469537735,
627
+ "learning_rate": 0.00025203703703703705,
628
+ "loss": 3.7946,
629
+ "step": 1640
630
+ },
631
+ {
632
+ "epoch": 0.5003390852234195,
633
+ "grad_norm": 0.2184811681509018,
634
+ "learning_rate": 0.0002483333333333333,
635
+ "loss": 3.7913,
636
+ "step": 1660
637
+ },
638
+ {
639
+ "epoch": 0.5063672669730992,
640
+ "grad_norm": 0.24770255386829376,
641
+ "learning_rate": 0.00024462962962962963,
642
+ "loss": 3.7902,
643
+ "step": 1680
644
+ },
645
+ {
646
+ "epoch": 0.512395448722779,
647
+ "grad_norm": 0.2306816428899765,
648
+ "learning_rate": 0.00024092592592592593,
649
+ "loss": 3.7775,
650
+ "step": 1700
651
+ },
652
+ {
653
+ "epoch": 0.5184236304724588,
654
+ "grad_norm": 0.2789316475391388,
655
+ "learning_rate": 0.00023722222222222222,
656
+ "loss": 3.7774,
657
+ "step": 1720
658
+ },
659
+ {
660
+ "epoch": 0.5244518122221385,
661
+ "grad_norm": 0.2576800286769867,
662
+ "learning_rate": 0.0002335185185185185,
663
+ "loss": 3.7773,
664
+ "step": 1740
665
+ },
666
+ {
667
+ "epoch": 0.5304799939718182,
668
+ "grad_norm": 0.2184872031211853,
669
+ "learning_rate": 0.0002298148148148148,
670
+ "loss": 3.7739,
671
+ "step": 1760
672
+ },
673
+ {
674
+ "epoch": 0.536508175721498,
675
+ "grad_norm": 0.27034640312194824,
676
+ "learning_rate": 0.00022611111111111112,
677
+ "loss": 3.7632,
678
+ "step": 1780
679
+ },
680
+ {
681
+ "epoch": 0.5425363574711778,
682
+ "grad_norm": 0.2661360800266266,
683
+ "learning_rate": 0.0002224074074074074,
684
+ "loss": 3.7668,
685
+ "step": 1800
686
+ },
687
+ {
688
+ "epoch": 0.5425363574711778,
689
+ "eval_loss": 3.7104278931619175,
690
+ "eval_perplexity": 40.871291331988935,
691
+ "eval_runtime": 112.585,
692
+ "eval_samples_per_second": 14.736,
693
+ "eval_steps_per_second": 0.231,
694
+ "step": 1800
695
+ },
696
+ {
697
+ "epoch": 0.5485645392208575,
698
+ "grad_norm": 0.2934229373931885,
699
+ "learning_rate": 0.0002187037037037037,
700
+ "loss": 3.7645,
701
+ "step": 1820
702
+ },
703
+ {
704
+ "epoch": 0.5545927209705372,
705
+ "grad_norm": 0.24501343071460724,
706
+ "learning_rate": 0.000215,
707
+ "loss": 3.7559,
708
+ "step": 1840
709
+ },
710
+ {
711
+ "epoch": 0.560620902720217,
712
+ "grad_norm": 0.21381346881389618,
713
+ "learning_rate": 0.00021129629629629629,
714
+ "loss": 3.7482,
715
+ "step": 1860
716
+ },
717
+ {
718
+ "epoch": 0.5666490844698968,
719
+ "grad_norm": 0.2350403219461441,
720
+ "learning_rate": 0.0002075925925925926,
721
+ "loss": 3.7488,
722
+ "step": 1880
723
+ },
724
+ {
725
+ "epoch": 0.5726772662195765,
726
+ "grad_norm": 0.23535749316215515,
727
+ "learning_rate": 0.0002038888888888889,
728
+ "loss": 3.7409,
729
+ "step": 1900
730
+ },
731
+ {
732
+ "epoch": 0.5787054479692563,
733
+ "grad_norm": 0.22763009369373322,
734
+ "learning_rate": 0.0002001851851851852,
735
+ "loss": 3.7431,
736
+ "step": 1920
737
+ },
738
+ {
739
+ "epoch": 0.584733629718936,
740
+ "grad_norm": 0.22279077768325806,
741
+ "learning_rate": 0.00019648148148148148,
742
+ "loss": 3.7373,
743
+ "step": 1940
744
+ },
745
+ {
746
+ "epoch": 0.5907618114686157,
747
+ "grad_norm": 0.19868987798690796,
748
+ "learning_rate": 0.00019277777777777777,
749
+ "loss": 3.7385,
750
+ "step": 1960
751
+ },
752
+ {
753
+ "epoch": 0.5967899932182955,
754
+ "grad_norm": 0.22577470541000366,
755
+ "learning_rate": 0.00018907407407407406,
756
+ "loss": 3.7266,
757
+ "step": 1980
758
+ },
759
+ {
760
+ "epoch": 0.6028181749679753,
761
+ "grad_norm": 0.23861481249332428,
762
+ "learning_rate": 0.00018537037037037038,
763
+ "loss": 3.7288,
764
+ "step": 2000
765
+ },
766
+ {
767
+ "epoch": 0.6088463567176551,
768
+ "grad_norm": 0.2368811070919037,
769
+ "learning_rate": 0.00018166666666666667,
770
+ "loss": 3.7251,
771
+ "step": 2020
772
+ },
773
+ {
774
+ "epoch": 0.6148745384673348,
775
+ "grad_norm": 0.23746469616889954,
776
+ "learning_rate": 0.00017796296296296296,
777
+ "loss": 3.7194,
778
+ "step": 2040
779
+ },
780
+ {
781
+ "epoch": 0.6209027202170145,
782
+ "grad_norm": 0.19934986531734467,
783
+ "learning_rate": 0.00017425925925925926,
784
+ "loss": 3.7203,
785
+ "step": 2060
786
+ },
787
+ {
788
+ "epoch": 0.6269309019666943,
789
+ "grad_norm": 0.22287386655807495,
790
+ "learning_rate": 0.00017055555555555555,
791
+ "loss": 3.7182,
792
+ "step": 2080
793
+ },
794
+ {
795
+ "epoch": 0.632959083716374,
796
+ "grad_norm": 0.21625646948814392,
797
+ "learning_rate": 0.00016685185185185187,
798
+ "loss": 3.7133,
799
+ "step": 2100
800
+ },
801
+ {
802
+ "epoch": 0.632959083716374,
803
+ "eval_loss": 3.661569493497066,
804
+ "eval_perplexity": 38.922383385070816,
805
+ "eval_runtime": 112.8832,
806
+ "eval_samples_per_second": 14.697,
807
+ "eval_steps_per_second": 0.23,
808
+ "step": 2100
809
+ },
810
+ {
811
+ "epoch": 0.6389872654660538,
812
+ "grad_norm": 0.2128862887620926,
813
+ "learning_rate": 0.00016314814814814816,
814
+ "loss": 3.7123,
815
+ "step": 2120
816
+ },
817
+ {
818
+ "epoch": 0.6450154472157336,
819
+ "grad_norm": 0.22579094767570496,
820
+ "learning_rate": 0.00015944444444444445,
821
+ "loss": 3.7103,
822
+ "step": 2140
823
+ },
824
+ {
825
+ "epoch": 0.6510436289654133,
826
+ "grad_norm": 0.19634632766246796,
827
+ "learning_rate": 0.00015574074074074074,
828
+ "loss": 3.7077,
829
+ "step": 2160
830
+ },
831
+ {
832
+ "epoch": 0.657071810715093,
833
+ "grad_norm": 0.21508827805519104,
834
+ "learning_rate": 0.00015203703703703703,
835
+ "loss": 3.7071,
836
+ "step": 2180
837
+ },
838
+ {
839
+ "epoch": 0.6630999924647728,
840
+ "grad_norm": 0.23243261873722076,
841
+ "learning_rate": 0.00014833333333333335,
842
+ "loss": 3.7035,
843
+ "step": 2200
844
+ },
845
+ {
846
+ "epoch": 0.6691281742144526,
847
+ "grad_norm": 0.22189666330814362,
848
+ "learning_rate": 0.00014462962962962964,
849
+ "loss": 3.7011,
850
+ "step": 2220
851
+ },
852
+ {
853
+ "epoch": 0.6751563559641324,
854
+ "grad_norm": 0.19845160841941833,
855
+ "learning_rate": 0.00014092592592592594,
856
+ "loss": 3.6996,
857
+ "step": 2240
858
+ },
859
+ {
860
+ "epoch": 0.681184537713812,
861
+ "grad_norm": 0.1785988211631775,
862
+ "learning_rate": 0.00013722222222222223,
863
+ "loss": 3.6866,
864
+ "step": 2260
865
+ },
866
+ {
867
+ "epoch": 0.6872127194634918,
868
+ "grad_norm": 0.2001648247241974,
869
+ "learning_rate": 0.00013351851851851852,
870
+ "loss": 3.6939,
871
+ "step": 2280
872
+ },
873
+ {
874
+ "epoch": 0.6932409012131716,
875
+ "grad_norm": 0.21330291032791138,
876
+ "learning_rate": 0.00012981481481481484,
877
+ "loss": 3.6868,
878
+ "step": 2300
879
+ },
880
+ {
881
+ "epoch": 0.6992690829628513,
882
+ "grad_norm": 0.23209834098815918,
883
+ "learning_rate": 0.00012611111111111113,
884
+ "loss": 3.6803,
885
+ "step": 2320
886
+ },
887
+ {
888
+ "epoch": 0.7052972647125311,
889
+ "grad_norm": 0.18792849779129028,
890
+ "learning_rate": 0.00012240740740740742,
891
+ "loss": 3.6824,
892
+ "step": 2340
893
+ },
894
+ {
895
+ "epoch": 0.7113254464622109,
896
+ "grad_norm": 0.19127227365970612,
897
+ "learning_rate": 0.0001187037037037037,
898
+ "loss": 3.6821,
899
+ "step": 2360
900
+ },
901
+ {
902
+ "epoch": 0.7173536282118906,
903
+ "grad_norm": 0.21657773852348328,
904
+ "learning_rate": 0.000115,
905
+ "loss": 3.6727,
906
+ "step": 2380
907
+ },
908
+ {
909
+ "epoch": 0.7233818099615703,
910
+ "grad_norm": 0.1759604662656784,
911
+ "learning_rate": 0.0001112962962962963,
912
+ "loss": 3.6809,
913
+ "step": 2400
914
+ },
915
+ {
916
+ "epoch": 0.7233818099615703,
917
+ "eval_loss": 3.62554258938625,
918
+ "eval_perplexity": 37.54508920080861,
919
+ "eval_runtime": 112.4817,
920
+ "eval_samples_per_second": 14.749,
921
+ "eval_steps_per_second": 0.231,
922
+ "step": 2400
923
+ },
924
+ {
925
+ "epoch": 0.7294099917112501,
926
+ "grad_norm": 0.20114636421203613,
927
+ "learning_rate": 0.00010759259259259259,
928
+ "loss": 3.6772,
929
+ "step": 2420
930
+ },
931
+ {
932
+ "epoch": 0.7354381734609299,
933
+ "grad_norm": 0.20904237031936646,
934
+ "learning_rate": 0.00010388888888888889,
935
+ "loss": 3.6754,
936
+ "step": 2440
937
+ },
938
+ {
939
+ "epoch": 0.7414663552106096,
940
+ "grad_norm": 0.18807630240917206,
941
+ "learning_rate": 0.00010018518518518518,
942
+ "loss": 3.6756,
943
+ "step": 2460
944
+ },
945
+ {
946
+ "epoch": 0.7474945369602893,
947
+ "grad_norm": 0.17234675586223602,
948
+ "learning_rate": 9.648148148148149e-05,
949
+ "loss": 3.6686,
950
+ "step": 2480
951
+ },
952
+ {
953
+ "epoch": 0.7535227187099691,
954
+ "grad_norm": 0.18095088005065918,
955
+ "learning_rate": 9.277777777777778e-05,
956
+ "loss": 3.6691,
957
+ "step": 2500
958
+ },
959
+ {
960
+ "epoch": 0.7595509004596489,
961
+ "grad_norm": 0.19775047898292542,
962
+ "learning_rate": 8.907407407407407e-05,
963
+ "loss": 3.6679,
964
+ "step": 2520
965
+ },
966
+ {
967
+ "epoch": 0.7655790822093286,
968
+ "grad_norm": 0.1963493674993515,
969
+ "learning_rate": 8.537037037037038e-05,
970
+ "loss": 3.668,
971
+ "step": 2540
972
+ },
973
+ {
974
+ "epoch": 0.7716072639590084,
975
+ "grad_norm": 0.16776752471923828,
976
+ "learning_rate": 8.166666666666667e-05,
977
+ "loss": 3.663,
978
+ "step": 2560
979
+ },
980
+ {
981
+ "epoch": 0.7776354457086881,
982
+ "grad_norm": 0.18634021282196045,
983
+ "learning_rate": 7.796296296296296e-05,
984
+ "loss": 3.6616,
985
+ "step": 2580
986
+ },
987
+ {
988
+ "epoch": 0.7836636274583678,
989
+ "grad_norm": 0.1853896975517273,
990
+ "learning_rate": 7.425925925925927e-05,
991
+ "loss": 3.6554,
992
+ "step": 2600
993
+ },
994
+ {
995
+ "epoch": 0.7896918092080476,
996
+ "grad_norm": 0.19131463766098022,
997
+ "learning_rate": 7.055555555555556e-05,
998
+ "loss": 3.6617,
999
+ "step": 2620
1000
+ },
1001
+ {
1002
+ "epoch": 0.7957199909577274,
1003
+ "grad_norm": 0.1861460953950882,
1004
+ "learning_rate": 6.685185185185186e-05,
1005
+ "loss": 3.6589,
1006
+ "step": 2640
1007
+ },
1008
+ {
1009
+ "epoch": 0.8017481727074072,
1010
+ "grad_norm": 0.18328429758548737,
1011
+ "learning_rate": 6.314814814814815e-05,
1012
+ "loss": 3.6581,
1013
+ "step": 2660
1014
+ },
1015
+ {
1016
+ "epoch": 0.8077763544570868,
1017
+ "grad_norm": 0.1829613447189331,
1018
+ "learning_rate": 5.9444444444444445e-05,
1019
+ "loss": 3.6508,
1020
+ "step": 2680
1021
+ },
1022
+ {
1023
+ "epoch": 0.8138045362067666,
1024
+ "grad_norm": 0.17789465188980103,
1025
+ "learning_rate": 5.5740740740740744e-05,
1026
+ "loss": 3.6536,
1027
+ "step": 2700
1028
+ },
1029
+ {
1030
+ "epoch": 0.8138045362067666,
1031
+ "eval_loss": 3.600645640400387,
1032
+ "eval_perplexity": 36.62187137207453,
1033
+ "eval_runtime": 112.2753,
1034
+ "eval_samples_per_second": 14.776,
1035
+ "eval_steps_per_second": 0.232,
1036
+ "step": 2700
1037
+ }
1038
+ ],
1039
+ "logging_steps": 20,
1040
+ "max_steps": 3000,
1041
+ "num_input_tokens_seen": 0,
1042
+ "num_train_epochs": 1,
1043
+ "save_steps": 300,
1044
+ "stateful_callbacks": {
1045
+ "TrainerControl": {
1046
+ "args": {
1047
+ "should_epoch_stop": false,
1048
+ "should_evaluate": false,
1049
+ "should_log": false,
1050
+ "should_save": true,
1051
+ "should_training_stop": false
1052
+ },
1053
+ "attributes": {}
1054
+ }
1055
+ },
1056
+ "total_flos": 1.4440968002469888e+18,
1057
+ "train_batch_size": 64,
1058
+ "trial_name": null,
1059
+ "trial_params": null
1060
+ }
checkpoint-2700/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08d76f8b5b85f05fb7caa15e408143d3ba0686e42d70879850259702897d08d2
3
+ size 5905
checkpoint-2700/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-3000/config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "SharedSpaceDecoderForCausalLM"
4
+ ],
5
+ "attention_backend": "flash_attention_2",
6
+ "attention_bias": false,
7
+ "attention_dropout_prob": 0.1,
8
+ "bos_token_id": 50256,
9
+ "classifier_dropout": null,
10
+ "dtype": "float32",
11
+ "eos_token_id": 50256,
12
+ "ffn_decompose": false,
13
+ "ffn_rank": null,
14
+ "hidden_dropout_prob": 0.1,
15
+ "hidden_size": 768,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 2048,
18
+ "kv_shared_dim": null,
19
+ "layer_norm_eps": 1e-12,
20
+ "max_position_embeddings": 1024,
21
+ "model_type": "shared_subspace_decoder",
22
+ "nope_dims": 32,
23
+ "norm_type": "rmsnorm",
24
+ "num_attention_heads": 12,
25
+ "num_dense_layers": 0,
26
+ "num_hidden_layers": 12,
27
+ "o_shared_dim": null,
28
+ "pad_token_id": 50256,
29
+ "q_shared_dim": null,
30
+ "qk_private_dim": 64,
31
+ "rms_norm_eps": 1e-06,
32
+ "rope_dims": 32,
33
+ "rope_scaling": {
34
+ "factor": 2.0,
35
+ "type": "linear"
36
+ },
37
+ "rope_theta": 10000.0,
38
+ "transformers_version": "4.56.0",
39
+ "vo_private_dim": 64,
40
+ "vocab_rank": null,
41
+ "vocab_size": 50257,
42
+ "vocab_subspace": false
43
+ }
checkpoint-3000/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-3000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6919065c808b2b25bd22da1e5d8cd5f6ee1111b1996e2c9ef132a5bd57e383d
3
+ size 988989899
checkpoint-3000/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02c749023ca02a2260541bcbe926f5ccb8aaea29a6b8d40e44cffcc695c4bdf0
3
+ size 494483579
checkpoint-3000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0cd23626013be2cf1b8f351eb1880911bc653b0d6a7fc40aa3f0c07b9f92b902
3
+ size 14645
checkpoint-3000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de993877decafa3dbec3a2190f9cbfc7ce6efc997a47f7abd25897f127fbf6ba
3
+ size 1465
checkpoint-3000/special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "unk_token": "<|endoftext|>"
6
+ }
checkpoint-3000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-3000/tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ }
12
+ },
13
+ "bos_token": "<|endoftext|>",
14
+ "clean_up_tokenization_spaces": false,
15
+ "eos_token": "<|endoftext|>",
16
+ "extra_special_tokens": {},
17
+ "model_max_length": 1024,
18
+ "pad_token": "<|endoftext|>",
19
+ "tokenizer_class": "GPT2Tokenizer",
20
+ "unk_token": "<|endoftext|>"
21
+ }
checkpoint-3000/trainer_state.json ADDED
@@ -0,0 +1,1174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 3000,
3
+ "best_metric": 3.587533058962724,
4
+ "best_model_checkpoint": "checkpoints/gpt-2_seq1024_mla0-0-0/checkpoint-3000",
5
+ "epoch": 0.9042272624519629,
6
+ "eval_steps": 300,
7
+ "global_step": 3000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.006028181749679753,
14
+ "grad_norm": 2.0441031455993652,
15
+ "learning_rate": 3.166666666666667e-05,
16
+ "loss": 10.3921,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.012056363499359506,
21
+ "grad_norm": 1.7166392803192139,
22
+ "learning_rate": 6.500000000000001e-05,
23
+ "loss": 9.241,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.01808454524903926,
28
+ "grad_norm": 0.7614754438400269,
29
+ "learning_rate": 9.833333333333333e-05,
30
+ "loss": 8.2219,
31
+ "step": 60
32
+ },
33
+ {
34
+ "epoch": 0.02411272699871901,
35
+ "grad_norm": 0.46116286516189575,
36
+ "learning_rate": 0.00013166666666666665,
37
+ "loss": 7.5368,
38
+ "step": 80
39
+ },
40
+ {
41
+ "epoch": 0.030140908748398764,
42
+ "grad_norm": 1.6069883108139038,
43
+ "learning_rate": 0.000165,
44
+ "loss": 7.21,
45
+ "step": 100
46
+ },
47
+ {
48
+ "epoch": 0.03616909049807852,
49
+ "grad_norm": 1.2077178955078125,
50
+ "learning_rate": 0.00019833333333333335,
51
+ "loss": 6.8976,
52
+ "step": 120
53
+ },
54
+ {
55
+ "epoch": 0.04219727224775827,
56
+ "grad_norm": 0.642643928527832,
57
+ "learning_rate": 0.00023166666666666667,
58
+ "loss": 6.641,
59
+ "step": 140
60
+ },
61
+ {
62
+ "epoch": 0.04822545399743802,
63
+ "grad_norm": 0.7632517218589783,
64
+ "learning_rate": 0.00026500000000000004,
65
+ "loss": 6.4194,
66
+ "step": 160
67
+ },
68
+ {
69
+ "epoch": 0.05425363574711778,
70
+ "grad_norm": 1.2542935609817505,
71
+ "learning_rate": 0.00029833333333333334,
72
+ "loss": 6.2428,
73
+ "step": 180
74
+ },
75
+ {
76
+ "epoch": 0.06028181749679753,
77
+ "grad_norm": 1.1738799810409546,
78
+ "learning_rate": 0.0003316666666666667,
79
+ "loss": 6.1026,
80
+ "step": 200
81
+ },
82
+ {
83
+ "epoch": 0.06630999924647728,
84
+ "grad_norm": 0.46843603253364563,
85
+ "learning_rate": 0.000365,
86
+ "loss": 5.9586,
87
+ "step": 220
88
+ },
89
+ {
90
+ "epoch": 0.07233818099615703,
91
+ "grad_norm": 0.3949953019618988,
92
+ "learning_rate": 0.00039833333333333333,
93
+ "loss": 5.8304,
94
+ "step": 240
95
+ },
96
+ {
97
+ "epoch": 0.07836636274583679,
98
+ "grad_norm": 1.0866719484329224,
99
+ "learning_rate": 0.0004316666666666667,
100
+ "loss": 5.7185,
101
+ "step": 260
102
+ },
103
+ {
104
+ "epoch": 0.08439454449551655,
105
+ "grad_norm": 0.34839189052581787,
106
+ "learning_rate": 0.000465,
107
+ "loss": 5.6037,
108
+ "step": 280
109
+ },
110
+ {
111
+ "epoch": 0.09042272624519629,
112
+ "grad_norm": 0.5791395902633667,
113
+ "learning_rate": 0.0004983333333333334,
114
+ "loss": 5.5003,
115
+ "step": 300
116
+ },
117
+ {
118
+ "epoch": 0.09042272624519629,
119
+ "eval_loss": 5.428642945231348,
120
+ "eval_perplexity": 227.839844387096,
121
+ "eval_runtime": 113.1707,
122
+ "eval_samples_per_second": 14.659,
123
+ "eval_steps_per_second": 0.23,
124
+ "step": 300
125
+ },
126
+ {
127
+ "epoch": 0.09645090799487605,
128
+ "grad_norm": 0.532134473323822,
129
+ "learning_rate": 0.0004964814814814814,
130
+ "loss": 5.3805,
131
+ "step": 320
132
+ },
133
+ {
134
+ "epoch": 0.1024790897445558,
135
+ "grad_norm": 0.5876124501228333,
136
+ "learning_rate": 0.0004927777777777777,
137
+ "loss": 5.2775,
138
+ "step": 340
139
+ },
140
+ {
141
+ "epoch": 0.10850727149423556,
142
+ "grad_norm": 0.4140375256538391,
143
+ "learning_rate": 0.0004890740740740741,
144
+ "loss": 5.193,
145
+ "step": 360
146
+ },
147
+ {
148
+ "epoch": 0.1145354532439153,
149
+ "grad_norm": 0.5529780387878418,
150
+ "learning_rate": 0.0004853703703703704,
151
+ "loss": 5.0912,
152
+ "step": 380
153
+ },
154
+ {
155
+ "epoch": 0.12056363499359506,
156
+ "grad_norm": 0.3331276774406433,
157
+ "learning_rate": 0.0004816666666666667,
158
+ "loss": 5.0266,
159
+ "step": 400
160
+ },
161
+ {
162
+ "epoch": 0.1265918167432748,
163
+ "grad_norm": 0.6916081309318542,
164
+ "learning_rate": 0.00047796296296296297,
165
+ "loss": 4.9375,
166
+ "step": 420
167
+ },
168
+ {
169
+ "epoch": 0.13261999849295456,
170
+ "grad_norm": 0.31799590587615967,
171
+ "learning_rate": 0.0004742592592592593,
172
+ "loss": 4.9026,
173
+ "step": 440
174
+ },
175
+ {
176
+ "epoch": 0.1386481802426343,
177
+ "grad_norm": 0.5735257267951965,
178
+ "learning_rate": 0.00047055555555555555,
179
+ "loss": 4.8075,
180
+ "step": 460
181
+ },
182
+ {
183
+ "epoch": 0.14467636199231407,
184
+ "grad_norm": 0.5078163146972656,
185
+ "learning_rate": 0.00046685185185185187,
186
+ "loss": 4.759,
187
+ "step": 480
188
+ },
189
+ {
190
+ "epoch": 0.15070454374199382,
191
+ "grad_norm": 0.3884360194206238,
192
+ "learning_rate": 0.00046314814814814813,
193
+ "loss": 4.6706,
194
+ "step": 500
195
+ },
196
+ {
197
+ "epoch": 0.15673272549167358,
198
+ "grad_norm": 0.5591968297958374,
199
+ "learning_rate": 0.00045944444444444445,
200
+ "loss": 4.627,
201
+ "step": 520
202
+ },
203
+ {
204
+ "epoch": 0.16276090724135334,
205
+ "grad_norm": 0.29390859603881836,
206
+ "learning_rate": 0.0004557407407407407,
207
+ "loss": 4.5691,
208
+ "step": 540
209
+ },
210
+ {
211
+ "epoch": 0.1687890889910331,
212
+ "grad_norm": 0.3524036407470703,
213
+ "learning_rate": 0.00045203703703703703,
214
+ "loss": 4.5066,
215
+ "step": 560
216
+ },
217
+ {
218
+ "epoch": 0.17481727074071282,
219
+ "grad_norm": 0.4607946574687958,
220
+ "learning_rate": 0.0004483333333333333,
221
+ "loss": 4.4661,
222
+ "step": 580
223
+ },
224
+ {
225
+ "epoch": 0.18084545249039258,
226
+ "grad_norm": 0.2523379623889923,
227
+ "learning_rate": 0.00044462962962962967,
228
+ "loss": 4.4331,
229
+ "step": 600
230
+ },
231
+ {
232
+ "epoch": 0.18084545249039258,
233
+ "eval_loss": 4.357323692651888,
234
+ "eval_perplexity": 78.04797429242741,
235
+ "eval_runtime": 112.5783,
236
+ "eval_samples_per_second": 14.736,
237
+ "eval_steps_per_second": 0.231,
238
+ "step": 600
239
+ },
240
+ {
241
+ "epoch": 0.18687363424007233,
242
+ "grad_norm": 0.3953287601470947,
243
+ "learning_rate": 0.00044092592592592594,
244
+ "loss": 4.3877,
245
+ "step": 620
246
+ },
247
+ {
248
+ "epoch": 0.1929018159897521,
249
+ "grad_norm": 0.38012024760246277,
250
+ "learning_rate": 0.00043722222222222225,
251
+ "loss": 4.3619,
252
+ "step": 640
253
+ },
254
+ {
255
+ "epoch": 0.19892999773943185,
256
+ "grad_norm": 0.3135126233100891,
257
+ "learning_rate": 0.0004335185185185185,
258
+ "loss": 4.3442,
259
+ "step": 660
260
+ },
261
+ {
262
+ "epoch": 0.2049581794891116,
263
+ "grad_norm": 0.49422895908355713,
264
+ "learning_rate": 0.00042981481481481484,
265
+ "loss": 4.3045,
266
+ "step": 680
267
+ },
268
+ {
269
+ "epoch": 0.21098636123879136,
270
+ "grad_norm": 0.27019643783569336,
271
+ "learning_rate": 0.0004261111111111111,
272
+ "loss": 4.2808,
273
+ "step": 700
274
+ },
275
+ {
276
+ "epoch": 0.21701454298847112,
277
+ "grad_norm": 0.3935423791408539,
278
+ "learning_rate": 0.0004224074074074074,
279
+ "loss": 4.2467,
280
+ "step": 720
281
+ },
282
+ {
283
+ "epoch": 0.22304272473815084,
284
+ "grad_norm": 0.28830042481422424,
285
+ "learning_rate": 0.0004187037037037037,
286
+ "loss": 4.2274,
287
+ "step": 740
288
+ },
289
+ {
290
+ "epoch": 0.2290709064878306,
291
+ "grad_norm": 0.2877708673477173,
292
+ "learning_rate": 0.000415,
293
+ "loss": 4.2075,
294
+ "step": 760
295
+ },
296
+ {
297
+ "epoch": 0.23509908823751036,
298
+ "grad_norm": 0.3449930250644684,
299
+ "learning_rate": 0.00041129629629629627,
300
+ "loss": 4.1909,
301
+ "step": 780
302
+ },
303
+ {
304
+ "epoch": 0.2411272699871901,
305
+ "grad_norm": 0.326265424489975,
306
+ "learning_rate": 0.00040759259259259264,
307
+ "loss": 4.1684,
308
+ "step": 800
309
+ },
310
+ {
311
+ "epoch": 0.24715545173686987,
312
+ "grad_norm": 0.3821180760860443,
313
+ "learning_rate": 0.0004038888888888889,
314
+ "loss": 4.1513,
315
+ "step": 820
316
+ },
317
+ {
318
+ "epoch": 0.2531836334865496,
319
+ "grad_norm": 0.2768200635910034,
320
+ "learning_rate": 0.0004001851851851852,
321
+ "loss": 4.1317,
322
+ "step": 840
323
+ },
324
+ {
325
+ "epoch": 0.2592118152362294,
326
+ "grad_norm": 0.29724735021591187,
327
+ "learning_rate": 0.0003964814814814815,
328
+ "loss": 4.1171,
329
+ "step": 860
330
+ },
331
+ {
332
+ "epoch": 0.2652399969859091,
333
+ "grad_norm": 0.2814749479293823,
334
+ "learning_rate": 0.0003927777777777778,
335
+ "loss": 4.1049,
336
+ "step": 880
337
+ },
338
+ {
339
+ "epoch": 0.2712681787355889,
340
+ "grad_norm": 0.26310989260673523,
341
+ "learning_rate": 0.00038907407407407407,
342
+ "loss": 4.0858,
343
+ "step": 900
344
+ },
345
+ {
346
+ "epoch": 0.2712681787355889,
347
+ "eval_loss": 4.030442206068737,
348
+ "eval_perplexity": 56.285795665128816,
349
+ "eval_runtime": 112.6912,
350
+ "eval_samples_per_second": 14.722,
351
+ "eval_steps_per_second": 0.231,
352
+ "step": 900
353
+ },
354
+ {
355
+ "epoch": 0.2772963604852686,
356
+ "grad_norm": 0.3063352108001709,
357
+ "learning_rate": 0.0003853703703703704,
358
+ "loss": 4.0813,
359
+ "step": 920
360
+ },
361
+ {
362
+ "epoch": 0.2833245422349484,
363
+ "grad_norm": 0.4063352346420288,
364
+ "learning_rate": 0.00038166666666666666,
365
+ "loss": 4.0647,
366
+ "step": 940
367
+ },
368
+ {
369
+ "epoch": 0.28935272398462814,
370
+ "grad_norm": 0.2601182758808136,
371
+ "learning_rate": 0.000377962962962963,
372
+ "loss": 4.0447,
373
+ "step": 960
374
+ },
375
+ {
376
+ "epoch": 0.29538090573430786,
377
+ "grad_norm": 0.3153649568557739,
378
+ "learning_rate": 0.00037425925925925924,
379
+ "loss": 4.0417,
380
+ "step": 980
381
+ },
382
+ {
383
+ "epoch": 0.30140908748398765,
384
+ "grad_norm": 0.26004716753959656,
385
+ "learning_rate": 0.0003705555555555556,
386
+ "loss": 4.0279,
387
+ "step": 1000
388
+ },
389
+ {
390
+ "epoch": 0.3074372692336674,
391
+ "grad_norm": 0.43524181842803955,
392
+ "learning_rate": 0.0003668518518518519,
393
+ "loss": 4.0152,
394
+ "step": 1020
395
+ },
396
+ {
397
+ "epoch": 0.31346545098334716,
398
+ "grad_norm": 0.2601913511753082,
399
+ "learning_rate": 0.0003631481481481482,
400
+ "loss": 4.0026,
401
+ "step": 1040
402
+ },
403
+ {
404
+ "epoch": 0.3194936327330269,
405
+ "grad_norm": 0.25712424516677856,
406
+ "learning_rate": 0.00035944444444444446,
407
+ "loss": 3.9933,
408
+ "step": 1060
409
+ },
410
+ {
411
+ "epoch": 0.3255218144827067,
412
+ "grad_norm": 0.3313097357749939,
413
+ "learning_rate": 0.0003557407407407408,
414
+ "loss": 3.9842,
415
+ "step": 1080
416
+ },
417
+ {
418
+ "epoch": 0.3315499962323864,
419
+ "grad_norm": 0.238124281167984,
420
+ "learning_rate": 0.00035203703703703704,
421
+ "loss": 3.9745,
422
+ "step": 1100
423
+ },
424
+ {
425
+ "epoch": 0.3375781779820662,
426
+ "grad_norm": 0.28039032220840454,
427
+ "learning_rate": 0.00034833333333333336,
428
+ "loss": 3.9588,
429
+ "step": 1120
430
+ },
431
+ {
432
+ "epoch": 0.3436063597317459,
433
+ "grad_norm": 0.3661600649356842,
434
+ "learning_rate": 0.0003446296296296296,
435
+ "loss": 3.9532,
436
+ "step": 1140
437
+ },
438
+ {
439
+ "epoch": 0.34963454148142564,
440
+ "grad_norm": 0.22854666411876678,
441
+ "learning_rate": 0.0003409259259259259,
442
+ "loss": 3.9392,
443
+ "step": 1160
444
+ },
445
+ {
446
+ "epoch": 0.3556627232311054,
447
+ "grad_norm": 0.2930833697319031,
448
+ "learning_rate": 0.0003372222222222222,
449
+ "loss": 3.9312,
450
+ "step": 1180
451
+ },
452
+ {
453
+ "epoch": 0.36169090498078516,
454
+ "grad_norm": 0.25500425696372986,
455
+ "learning_rate": 0.0003335185185185185,
456
+ "loss": 3.9295,
457
+ "step": 1200
458
+ },
459
+ {
460
+ "epoch": 0.36169090498078516,
461
+ "eval_loss": 3.874079793811062,
462
+ "eval_perplexity": 48.13838066773452,
463
+ "eval_runtime": 112.5895,
464
+ "eval_samples_per_second": 14.735,
465
+ "eval_steps_per_second": 0.231,
466
+ "step": 1200
467
+ },
468
+ {
469
+ "epoch": 0.36771908673046494,
470
+ "grad_norm": 0.27924397587776184,
471
+ "learning_rate": 0.00032981481481481485,
472
+ "loss": 3.9157,
473
+ "step": 1220
474
+ },
475
+ {
476
+ "epoch": 0.37374726848014467,
477
+ "grad_norm": 0.2649654448032379,
478
+ "learning_rate": 0.0003261111111111111,
479
+ "loss": 3.9136,
480
+ "step": 1240
481
+ },
482
+ {
483
+ "epoch": 0.37977545022982445,
484
+ "grad_norm": 0.27920904755592346,
485
+ "learning_rate": 0.00032240740740740743,
486
+ "loss": 3.899,
487
+ "step": 1260
488
+ },
489
+ {
490
+ "epoch": 0.3858036319795042,
491
+ "grad_norm": 0.238219752907753,
492
+ "learning_rate": 0.0003187037037037037,
493
+ "loss": 3.8965,
494
+ "step": 1280
495
+ },
496
+ {
497
+ "epoch": 0.3918318137291839,
498
+ "grad_norm": 0.2615567743778229,
499
+ "learning_rate": 0.000315,
500
+ "loss": 3.8877,
501
+ "step": 1300
502
+ },
503
+ {
504
+ "epoch": 0.3978599954788637,
505
+ "grad_norm": 0.266513466835022,
506
+ "learning_rate": 0.0003112962962962963,
507
+ "loss": 3.8806,
508
+ "step": 1320
509
+ },
510
+ {
511
+ "epoch": 0.4038881772285434,
512
+ "grad_norm": 0.3212108314037323,
513
+ "learning_rate": 0.0003075925925925926,
514
+ "loss": 3.8774,
515
+ "step": 1340
516
+ },
517
+ {
518
+ "epoch": 0.4099163589782232,
519
+ "grad_norm": 0.231920063495636,
520
+ "learning_rate": 0.00030388888888888886,
521
+ "loss": 3.8707,
522
+ "step": 1360
523
+ },
524
+ {
525
+ "epoch": 0.41594454072790293,
526
+ "grad_norm": 0.27022039890289307,
527
+ "learning_rate": 0.0003001851851851852,
528
+ "loss": 3.863,
529
+ "step": 1380
530
+ },
531
+ {
532
+ "epoch": 0.4219727224775827,
533
+ "grad_norm": 0.243785560131073,
534
+ "learning_rate": 0.00029648148148148144,
535
+ "loss": 3.8562,
536
+ "step": 1400
537
+ },
538
+ {
539
+ "epoch": 0.42800090422726245,
540
+ "grad_norm": 0.2512820065021515,
541
+ "learning_rate": 0.0002927777777777778,
542
+ "loss": 3.8541,
543
+ "step": 1420
544
+ },
545
+ {
546
+ "epoch": 0.43402908597694223,
547
+ "grad_norm": 0.25888124108314514,
548
+ "learning_rate": 0.0002890740740740741,
549
+ "loss": 3.8421,
550
+ "step": 1440
551
+ },
552
+ {
553
+ "epoch": 0.44005726772662196,
554
+ "grad_norm": 0.29554685950279236,
555
+ "learning_rate": 0.0002853703703703704,
556
+ "loss": 3.8468,
557
+ "step": 1460
558
+ },
559
+ {
560
+ "epoch": 0.4460854494763017,
561
+ "grad_norm": 0.2608514130115509,
562
+ "learning_rate": 0.00028166666666666666,
563
+ "loss": 3.837,
564
+ "step": 1480
565
+ },
566
+ {
567
+ "epoch": 0.45211363122598147,
568
+ "grad_norm": 0.2781771123409271,
569
+ "learning_rate": 0.000277962962962963,
570
+ "loss": 3.8327,
571
+ "step": 1500
572
+ },
573
+ {
574
+ "epoch": 0.45211363122598147,
575
+ "eval_loss": 3.776173532413324,
576
+ "eval_perplexity": 43.6487014373908,
577
+ "eval_runtime": 112.7037,
578
+ "eval_samples_per_second": 14.72,
579
+ "eval_steps_per_second": 0.231,
580
+ "step": 1500
581
+ },
582
+ {
583
+ "epoch": 0.4581418129756612,
584
+ "grad_norm": 0.27767959237098694,
585
+ "learning_rate": 0.00027425925925925925,
586
+ "loss": 3.8264,
587
+ "step": 1520
588
+ },
589
+ {
590
+ "epoch": 0.464169994725341,
591
+ "grad_norm": 0.24143943190574646,
592
+ "learning_rate": 0.00027055555555555557,
593
+ "loss": 3.8224,
594
+ "step": 1540
595
+ },
596
+ {
597
+ "epoch": 0.4701981764750207,
598
+ "grad_norm": 0.23281621932983398,
599
+ "learning_rate": 0.00026685185185185183,
600
+ "loss": 3.8114,
601
+ "step": 1560
602
+ },
603
+ {
604
+ "epoch": 0.4762263582247005,
605
+ "grad_norm": 0.23552870750427246,
606
+ "learning_rate": 0.00026314814814814815,
607
+ "loss": 3.8086,
608
+ "step": 1580
609
+ },
610
+ {
611
+ "epoch": 0.4822545399743802,
612
+ "grad_norm": 0.25271207094192505,
613
+ "learning_rate": 0.0002594444444444444,
614
+ "loss": 3.7954,
615
+ "step": 1600
616
+ },
617
+ {
618
+ "epoch": 0.48828272172405995,
619
+ "grad_norm": 0.2609969675540924,
620
+ "learning_rate": 0.0002557407407407408,
621
+ "loss": 3.8052,
622
+ "step": 1620
623
+ },
624
+ {
625
+ "epoch": 0.49431090347373974,
626
+ "grad_norm": 0.290229469537735,
627
+ "learning_rate": 0.00025203703703703705,
628
+ "loss": 3.7946,
629
+ "step": 1640
630
+ },
631
+ {
632
+ "epoch": 0.5003390852234195,
633
+ "grad_norm": 0.2184811681509018,
634
+ "learning_rate": 0.0002483333333333333,
635
+ "loss": 3.7913,
636
+ "step": 1660
637
+ },
638
+ {
639
+ "epoch": 0.5063672669730992,
640
+ "grad_norm": 0.24770255386829376,
641
+ "learning_rate": 0.00024462962962962963,
642
+ "loss": 3.7902,
643
+ "step": 1680
644
+ },
645
+ {
646
+ "epoch": 0.512395448722779,
647
+ "grad_norm": 0.2306816428899765,
648
+ "learning_rate": 0.00024092592592592593,
649
+ "loss": 3.7775,
650
+ "step": 1700
651
+ },
652
+ {
653
+ "epoch": 0.5184236304724588,
654
+ "grad_norm": 0.2789316475391388,
655
+ "learning_rate": 0.00023722222222222222,
656
+ "loss": 3.7774,
657
+ "step": 1720
658
+ },
659
+ {
660
+ "epoch": 0.5244518122221385,
661
+ "grad_norm": 0.2576800286769867,
662
+ "learning_rate": 0.0002335185185185185,
663
+ "loss": 3.7773,
664
+ "step": 1740
665
+ },
666
+ {
667
+ "epoch": 0.5304799939718182,
668
+ "grad_norm": 0.2184872031211853,
669
+ "learning_rate": 0.0002298148148148148,
670
+ "loss": 3.7739,
671
+ "step": 1760
672
+ },
673
+ {
674
+ "epoch": 0.536508175721498,
675
+ "grad_norm": 0.27034640312194824,
676
+ "learning_rate": 0.00022611111111111112,
677
+ "loss": 3.7632,
678
+ "step": 1780
679
+ },
680
+ {
681
+ "epoch": 0.5425363574711778,
682
+ "grad_norm": 0.2661360800266266,
683
+ "learning_rate": 0.0002224074074074074,
684
+ "loss": 3.7668,
685
+ "step": 1800
686
+ },
687
+ {
688
+ "epoch": 0.5425363574711778,
689
+ "eval_loss": 3.7104278931619175,
690
+ "eval_perplexity": 40.871291331988935,
691
+ "eval_runtime": 112.585,
692
+ "eval_samples_per_second": 14.736,
693
+ "eval_steps_per_second": 0.231,
694
+ "step": 1800
695
+ },
696
+ {
697
+ "epoch": 0.5485645392208575,
698
+ "grad_norm": 0.2934229373931885,
699
+ "learning_rate": 0.0002187037037037037,
700
+ "loss": 3.7645,
701
+ "step": 1820
702
+ },
703
+ {
704
+ "epoch": 0.5545927209705372,
705
+ "grad_norm": 0.24501343071460724,
706
+ "learning_rate": 0.000215,
707
+ "loss": 3.7559,
708
+ "step": 1840
709
+ },
710
+ {
711
+ "epoch": 0.560620902720217,
712
+ "grad_norm": 0.21381346881389618,
713
+ "learning_rate": 0.00021129629629629629,
714
+ "loss": 3.7482,
715
+ "step": 1860
716
+ },
717
+ {
718
+ "epoch": 0.5666490844698968,
719
+ "grad_norm": 0.2350403219461441,
720
+ "learning_rate": 0.0002075925925925926,
721
+ "loss": 3.7488,
722
+ "step": 1880
723
+ },
724
+ {
725
+ "epoch": 0.5726772662195765,
726
+ "grad_norm": 0.23535749316215515,
727
+ "learning_rate": 0.0002038888888888889,
728
+ "loss": 3.7409,
729
+ "step": 1900
730
+ },
731
+ {
732
+ "epoch": 0.5787054479692563,
733
+ "grad_norm": 0.22763009369373322,
734
+ "learning_rate": 0.0002001851851851852,
735
+ "loss": 3.7431,
736
+ "step": 1920
737
+ },
738
+ {
739
+ "epoch": 0.584733629718936,
740
+ "grad_norm": 0.22279077768325806,
741
+ "learning_rate": 0.00019648148148148148,
742
+ "loss": 3.7373,
743
+ "step": 1940
744
+ },
745
+ {
746
+ "epoch": 0.5907618114686157,
747
+ "grad_norm": 0.19868987798690796,
748
+ "learning_rate": 0.00019277777777777777,
749
+ "loss": 3.7385,
750
+ "step": 1960
751
+ },
752
+ {
753
+ "epoch": 0.5967899932182955,
754
+ "grad_norm": 0.22577470541000366,
755
+ "learning_rate": 0.00018907407407407406,
756
+ "loss": 3.7266,
757
+ "step": 1980
758
+ },
759
+ {
760
+ "epoch": 0.6028181749679753,
761
+ "grad_norm": 0.23861481249332428,
762
+ "learning_rate": 0.00018537037037037038,
763
+ "loss": 3.7288,
764
+ "step": 2000
765
+ },
766
+ {
767
+ "epoch": 0.6088463567176551,
768
+ "grad_norm": 0.2368811070919037,
769
+ "learning_rate": 0.00018166666666666667,
770
+ "loss": 3.7251,
771
+ "step": 2020
772
+ },
773
+ {
774
+ "epoch": 0.6148745384673348,
775
+ "grad_norm": 0.23746469616889954,
776
+ "learning_rate": 0.00017796296296296296,
777
+ "loss": 3.7194,
778
+ "step": 2040
779
+ },
780
+ {
781
+ "epoch": 0.6209027202170145,
782
+ "grad_norm": 0.19934986531734467,
783
+ "learning_rate": 0.00017425925925925926,
784
+ "loss": 3.7203,
785
+ "step": 2060
786
+ },
787
+ {
788
+ "epoch": 0.6269309019666943,
789
+ "grad_norm": 0.22287386655807495,
790
+ "learning_rate": 0.00017055555555555555,
791
+ "loss": 3.7182,
792
+ "step": 2080
793
+ },
794
+ {
795
+ "epoch": 0.632959083716374,
796
+ "grad_norm": 0.21625646948814392,
797
+ "learning_rate": 0.00016685185185185187,
798
+ "loss": 3.7133,
799
+ "step": 2100
800
+ },
801
+ {
802
+ "epoch": 0.632959083716374,
803
+ "eval_loss": 3.661569493497066,
804
+ "eval_perplexity": 38.922383385070816,
805
+ "eval_runtime": 112.8832,
806
+ "eval_samples_per_second": 14.697,
807
+ "eval_steps_per_second": 0.23,
808
+ "step": 2100
809
+ },
810
+ {
811
+ "epoch": 0.6389872654660538,
812
+ "grad_norm": 0.2128862887620926,
813
+ "learning_rate": 0.00016314814814814816,
814
+ "loss": 3.7123,
815
+ "step": 2120
816
+ },
817
+ {
818
+ "epoch": 0.6450154472157336,
819
+ "grad_norm": 0.22579094767570496,
820
+ "learning_rate": 0.00015944444444444445,
821
+ "loss": 3.7103,
822
+ "step": 2140
823
+ },
824
+ {
825
+ "epoch": 0.6510436289654133,
826
+ "grad_norm": 0.19634632766246796,
827
+ "learning_rate": 0.00015574074074074074,
828
+ "loss": 3.7077,
829
+ "step": 2160
830
+ },
831
+ {
832
+ "epoch": 0.657071810715093,
833
+ "grad_norm": 0.21508827805519104,
834
+ "learning_rate": 0.00015203703703703703,
835
+ "loss": 3.7071,
836
+ "step": 2180
837
+ },
838
+ {
839
+ "epoch": 0.6630999924647728,
840
+ "grad_norm": 0.23243261873722076,
841
+ "learning_rate": 0.00014833333333333335,
842
+ "loss": 3.7035,
843
+ "step": 2200
844
+ },
845
+ {
846
+ "epoch": 0.6691281742144526,
847
+ "grad_norm": 0.22189666330814362,
848
+ "learning_rate": 0.00014462962962962964,
849
+ "loss": 3.7011,
850
+ "step": 2220
851
+ },
852
+ {
853
+ "epoch": 0.6751563559641324,
854
+ "grad_norm": 0.19845160841941833,
855
+ "learning_rate": 0.00014092592592592594,
856
+ "loss": 3.6996,
857
+ "step": 2240
858
+ },
859
+ {
860
+ "epoch": 0.681184537713812,
861
+ "grad_norm": 0.1785988211631775,
862
+ "learning_rate": 0.00013722222222222223,
863
+ "loss": 3.6866,
864
+ "step": 2260
865
+ },
866
+ {
867
+ "epoch": 0.6872127194634918,
868
+ "grad_norm": 0.2001648247241974,
869
+ "learning_rate": 0.00013351851851851852,
870
+ "loss": 3.6939,
871
+ "step": 2280
872
+ },
873
+ {
874
+ "epoch": 0.6932409012131716,
875
+ "grad_norm": 0.21330291032791138,
876
+ "learning_rate": 0.00012981481481481484,
877
+ "loss": 3.6868,
878
+ "step": 2300
879
+ },
880
+ {
881
+ "epoch": 0.6992690829628513,
882
+ "grad_norm": 0.23209834098815918,
883
+ "learning_rate": 0.00012611111111111113,
884
+ "loss": 3.6803,
885
+ "step": 2320
886
+ },
887
+ {
888
+ "epoch": 0.7052972647125311,
889
+ "grad_norm": 0.18792849779129028,
890
+ "learning_rate": 0.00012240740740740742,
891
+ "loss": 3.6824,
892
+ "step": 2340
893
+ },
894
+ {
895
+ "epoch": 0.7113254464622109,
896
+ "grad_norm": 0.19127227365970612,
897
+ "learning_rate": 0.0001187037037037037,
898
+ "loss": 3.6821,
899
+ "step": 2360
900
+ },
901
+ {
902
+ "epoch": 0.7173536282118906,
903
+ "grad_norm": 0.21657773852348328,
904
+ "learning_rate": 0.000115,
905
+ "loss": 3.6727,
906
+ "step": 2380
907
+ },
908
+ {
909
+ "epoch": 0.7233818099615703,
910
+ "grad_norm": 0.1759604662656784,
911
+ "learning_rate": 0.0001112962962962963,
912
+ "loss": 3.6809,
913
+ "step": 2400
914
+ },
915
+ {
916
+ "epoch": 0.7233818099615703,
917
+ "eval_loss": 3.62554258938625,
918
+ "eval_perplexity": 37.54508920080861,
919
+ "eval_runtime": 112.4817,
920
+ "eval_samples_per_second": 14.749,
921
+ "eval_steps_per_second": 0.231,
922
+ "step": 2400
923
+ },
924
+ {
925
+ "epoch": 0.7294099917112501,
926
+ "grad_norm": 0.20114636421203613,
927
+ "learning_rate": 0.00010759259259259259,
928
+ "loss": 3.6772,
929
+ "step": 2420
930
+ },
931
+ {
932
+ "epoch": 0.7354381734609299,
933
+ "grad_norm": 0.20904237031936646,
934
+ "learning_rate": 0.00010388888888888889,
935
+ "loss": 3.6754,
936
+ "step": 2440
937
+ },
938
+ {
939
+ "epoch": 0.7414663552106096,
940
+ "grad_norm": 0.18807630240917206,
941
+ "learning_rate": 0.00010018518518518518,
942
+ "loss": 3.6756,
943
+ "step": 2460
944
+ },
945
+ {
946
+ "epoch": 0.7474945369602893,
947
+ "grad_norm": 0.17234675586223602,
948
+ "learning_rate": 9.648148148148149e-05,
949
+ "loss": 3.6686,
950
+ "step": 2480
951
+ },
952
+ {
953
+ "epoch": 0.7535227187099691,
954
+ "grad_norm": 0.18095088005065918,
955
+ "learning_rate": 9.277777777777778e-05,
956
+ "loss": 3.6691,
957
+ "step": 2500
958
+ },
959
+ {
960
+ "epoch": 0.7595509004596489,
961
+ "grad_norm": 0.19775047898292542,
962
+ "learning_rate": 8.907407407407407e-05,
963
+ "loss": 3.6679,
964
+ "step": 2520
965
+ },
966
+ {
967
+ "epoch": 0.7655790822093286,
968
+ "grad_norm": 0.1963493674993515,
969
+ "learning_rate": 8.537037037037038e-05,
970
+ "loss": 3.668,
971
+ "step": 2540
972
+ },
973
+ {
974
+ "epoch": 0.7716072639590084,
975
+ "grad_norm": 0.16776752471923828,
976
+ "learning_rate": 8.166666666666667e-05,
977
+ "loss": 3.663,
978
+ "step": 2560
979
+ },
980
+ {
981
+ "epoch": 0.7776354457086881,
982
+ "grad_norm": 0.18634021282196045,
983
+ "learning_rate": 7.796296296296296e-05,
984
+ "loss": 3.6616,
985
+ "step": 2580
986
+ },
987
+ {
988
+ "epoch": 0.7836636274583678,
989
+ "grad_norm": 0.1853896975517273,
990
+ "learning_rate": 7.425925925925927e-05,
991
+ "loss": 3.6554,
992
+ "step": 2600
993
+ },
994
+ {
995
+ "epoch": 0.7896918092080476,
996
+ "grad_norm": 0.19131463766098022,
997
+ "learning_rate": 7.055555555555556e-05,
998
+ "loss": 3.6617,
999
+ "step": 2620
1000
+ },
1001
+ {
1002
+ "epoch": 0.7957199909577274,
1003
+ "grad_norm": 0.1861460953950882,
1004
+ "learning_rate": 6.685185185185186e-05,
1005
+ "loss": 3.6589,
1006
+ "step": 2640
1007
+ },
1008
+ {
1009
+ "epoch": 0.8017481727074072,
1010
+ "grad_norm": 0.18328429758548737,
1011
+ "learning_rate": 6.314814814814815e-05,
1012
+ "loss": 3.6581,
1013
+ "step": 2660
1014
+ },
1015
+ {
1016
+ "epoch": 0.8077763544570868,
1017
+ "grad_norm": 0.1829613447189331,
1018
+ "learning_rate": 5.9444444444444445e-05,
1019
+ "loss": 3.6508,
1020
+ "step": 2680
1021
+ },
1022
+ {
1023
+ "epoch": 0.8138045362067666,
1024
+ "grad_norm": 0.17789465188980103,
1025
+ "learning_rate": 5.5740740740740744e-05,
1026
+ "loss": 3.6536,
1027
+ "step": 2700
1028
+ },
1029
+ {
1030
+ "epoch": 0.8138045362067666,
1031
+ "eval_loss": 3.600645640400387,
1032
+ "eval_perplexity": 36.62187137207453,
1033
+ "eval_runtime": 112.2753,
1034
+ "eval_samples_per_second": 14.776,
1035
+ "eval_steps_per_second": 0.232,
1036
+ "step": 2700
1037
+ },
1038
+ {
1039
+ "epoch": 0.8198327179564464,
1040
+ "grad_norm": 0.16772860288619995,
1041
+ "learning_rate": 5.2037037037037035e-05,
1042
+ "loss": 3.6515,
1043
+ "step": 2720
1044
+ },
1045
+ {
1046
+ "epoch": 0.8258608997061262,
1047
+ "grad_norm": 0.16776631772518158,
1048
+ "learning_rate": 4.8333333333333334e-05,
1049
+ "loss": 3.6539,
1050
+ "step": 2740
1051
+ },
1052
+ {
1053
+ "epoch": 0.8318890814558059,
1054
+ "grad_norm": 0.1612899899482727,
1055
+ "learning_rate": 4.462962962962963e-05,
1056
+ "loss": 3.6494,
1057
+ "step": 2760
1058
+ },
1059
+ {
1060
+ "epoch": 0.8379172632054857,
1061
+ "grad_norm": 0.16370627284049988,
1062
+ "learning_rate": 4.092592592592593e-05,
1063
+ "loss": 3.648,
1064
+ "step": 2780
1065
+ },
1066
+ {
1067
+ "epoch": 0.8439454449551654,
1068
+ "grad_norm": 0.1611148715019226,
1069
+ "learning_rate": 3.722222222222222e-05,
1070
+ "loss": 3.6498,
1071
+ "step": 2800
1072
+ },
1073
+ {
1074
+ "epoch": 0.8499736267048451,
1075
+ "grad_norm": 0.16145454347133636,
1076
+ "learning_rate": 3.351851851851852e-05,
1077
+ "loss": 3.6518,
1078
+ "step": 2820
1079
+ },
1080
+ {
1081
+ "epoch": 0.8560018084545249,
1082
+ "grad_norm": 0.1517009735107422,
1083
+ "learning_rate": 2.9814814814814815e-05,
1084
+ "loss": 3.644,
1085
+ "step": 2840
1086
+ },
1087
+ {
1088
+ "epoch": 0.8620299902042047,
1089
+ "grad_norm": 0.1570337861776352,
1090
+ "learning_rate": 2.6111111111111114e-05,
1091
+ "loss": 3.6417,
1092
+ "step": 2860
1093
+ },
1094
+ {
1095
+ "epoch": 0.8680581719538845,
1096
+ "grad_norm": 0.16112424433231354,
1097
+ "learning_rate": 2.240740740740741e-05,
1098
+ "loss": 3.6459,
1099
+ "step": 2880
1100
+ },
1101
+ {
1102
+ "epoch": 0.8740863537035641,
1103
+ "grad_norm": 0.15278516709804535,
1104
+ "learning_rate": 1.8703703703703707e-05,
1105
+ "loss": 3.6408,
1106
+ "step": 2900
1107
+ },
1108
+ {
1109
+ "epoch": 0.8801145354532439,
1110
+ "grad_norm": 0.15267758071422577,
1111
+ "learning_rate": 1.5e-05,
1112
+ "loss": 3.6452,
1113
+ "step": 2920
1114
+ },
1115
+ {
1116
+ "epoch": 0.8861427172029237,
1117
+ "grad_norm": 0.14383205771446228,
1118
+ "learning_rate": 1.1296296296296295e-05,
1119
+ "loss": 3.6364,
1120
+ "step": 2940
1121
+ },
1122
+ {
1123
+ "epoch": 0.8921708989526034,
1124
+ "grad_norm": 0.14794571697711945,
1125
+ "learning_rate": 7.592592592592593e-06,
1126
+ "loss": 3.6325,
1127
+ "step": 2960
1128
+ },
1129
+ {
1130
+ "epoch": 0.8981990807022832,
1131
+ "grad_norm": 0.14148849248886108,
1132
+ "learning_rate": 3.888888888888889e-06,
1133
+ "loss": 3.64,
1134
+ "step": 2980
1135
+ },
1136
+ {
1137
+ "epoch": 0.9042272624519629,
1138
+ "grad_norm": 0.1426716446876526,
1139
+ "learning_rate": 1.8518518518518518e-07,
1140
+ "loss": 3.6443,
1141
+ "step": 3000
1142
+ },
1143
+ {
1144
+ "epoch": 0.9042272624519629,
1145
+ "eval_loss": 3.587533058962724,
1146
+ "eval_perplexity": 36.1447987636509,
1147
+ "eval_runtime": 112.2843,
1148
+ "eval_samples_per_second": 14.775,
1149
+ "eval_steps_per_second": 0.232,
1150
+ "step": 3000
1151
+ }
1152
+ ],
1153
+ "logging_steps": 20,
1154
+ "max_steps": 3000,
1155
+ "num_input_tokens_seen": 0,
1156
+ "num_train_epochs": 1,
1157
+ "save_steps": 300,
1158
+ "stateful_callbacks": {
1159
+ "TrainerControl": {
1160
+ "args": {
1161
+ "should_epoch_stop": false,
1162
+ "should_evaluate": false,
1163
+ "should_log": false,
1164
+ "should_save": true,
1165
+ "should_training_stop": true
1166
+ },
1167
+ "attributes": {}
1168
+ }
1169
+ },
1170
+ "total_flos": 1.604552000274432e+18,
1171
+ "train_batch_size": 64,
1172
+ "trial_name": null,
1173
+ "trial_params": null
1174
+ }
checkpoint-3000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08d76f8b5b85f05fb7caa15e408143d3ba0686e42d70879850259702897d08d2
3
+ size 5905
checkpoint-3000/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
full_config.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "shorthand": "model.768.lyr.12 - seqlen.1024 - mla.0.0.0 - ah12.64 - rd32.32",
3
+ "notes": "GPT-2 run with essentially MHA.",
4
+ "model": {
5
+ "hidden_size": 768,
6
+ "num_hidden_layers": 12,
7
+ "intermediate_size": 2048,
8
+ "vocab_size": 50257,
9
+ "tie_word_embeddings": true,
10
+ "max_position_embeddings": 1024,
11
+ "norm_type": "rmsnorm",
12
+ "layer_norm_eps": 1e-12,
13
+ "rms_norm_eps": 1e-06,
14
+ "num_dense_layers": 0,
15
+ "num_attention_heads": 12,
16
+ "q_shared_dim": null,
17
+ "kv_shared_dim": null,
18
+ "o_shared_dim": null,
19
+ "qk_private_dim": 64,
20
+ "vo_private_dim": 64,
21
+ "rope_dims": 32,
22
+ "nope_dims": 32,
23
+ "rope_theta": 10000.0,
24
+ "rope_scaling": {
25
+ "type": "linear",
26
+ "factor": 2.0
27
+ },
28
+ "attention_bias": false,
29
+ "attention_backend": "flash_attention_2",
30
+ "ffn_decompose": false,
31
+ "ffn_rank": null,
32
+ "vocab_subspace": false,
33
+ "vocab_rank": null,
34
+ "hidden_dropout_prob": 0.1,
35
+ "attention_dropout_prob": 0.1,
36
+ "classifier_dropout": null,
37
+ "initializer_range": 0.02
38
+ },
39
+ "pre_train": {
40
+ "wandb_project": "decoder-pretrain-c4",
41
+ "output_dir": "checkpoints/gpt-2_seq1024_mla0-0-0",
42
+ "seed": 42,
43
+ "logging_steps": 20,
44
+ "save_steps": 300,
45
+ "train_batch_size": 64,
46
+ "gradient_accumulation_steps": 16,
47
+ "learning_rate": 0.0005,
48
+ "num_train_steps": 3000,
49
+ "eval_steps": 300,
50
+ "weight_decay": 0.01,
51
+ "num_workers": 8,
52
+ "pin_memory": true,
53
+ "_comment_dataset": "Use preprocessed_dataset_path instead of dataset streaming",
54
+ "preprocessed_dataset_path": "/home/ubuntu/c4_en_pct0.02_seq1024/c4_en_pct0.02_seq1024/dataset",
55
+ "dataset_name": "allenai/c4",
56
+ "dataset_config": "en",
57
+ "dataset_subset_pct": 0.02,
58
+ "max_seq_length": 1024,
59
+ "eval_batch_size": 64,
60
+ "fp16": false,
61
+ "bf16": true,
62
+ "torch_compile": true,
63
+ "torch_compile_backend": "inductor",
64
+ "torch_compile_mode": "default",
65
+ "run_name": "117.88M - model.768.lyr.12 - seqlen.1024 - mla.0.0.0 - ah12.64 - rd32.32",
66
+ "run_id": "15ixey47",
67
+ "run_url": "https://wandb.ai/chrismccormick/decoder-pretrain-c4/runs/15ixey47",
68
+ "best_checkpoint": "checkpoints/gpt-2_seq1024_mla0-0-0/checkpoint-3000"
69
+ },
70
+ "stats": {
71
+ "total_elements": "117.88M"
72
+ }
73
+ }