Hugging Face's logo

prithivMLmods
/
Gpt2-Wikitext-9180

Text Generation
Transformers
Safetensors
English
gpt2
text-generation-inference
Model card Files
xet
Community
prithivMLmods commited on
Commit
6f426c0
·
verified ·
1 Parent(s): 2c7417f

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. checkpoint-1000/config.json +38 -0
  2. checkpoint-1000/generation_config.json +6 -0
  3. checkpoint-1000/model.safetensors +3 -0
  4. checkpoint-1000/optimizer.pt +3 -0
  5. checkpoint-1000/rng_state.pth +3 -0
  6. checkpoint-1000/scheduler.pt +3 -0
  7. checkpoint-1000/trainer_state.json +48 -0
  8. checkpoint-1000/training_args.bin +3 -0
  9. checkpoint-1500/config.json +38 -0
  10. checkpoint-1500/generation_config.json +6 -0
  11. checkpoint-1500/model.safetensors +3 -0
  12. checkpoint-1500/optimizer.pt +3 -0
  13. checkpoint-1500/rng_state.pth +3 -0
  14. checkpoint-1500/scheduler.pt +3 -0
  15. checkpoint-1500/trainer_state.json +55 -0
  16. checkpoint-1500/training_args.bin +3 -0
  17. checkpoint-2000/config.json +38 -0
  18. checkpoint-2000/generation_config.json +6 -0
  19. checkpoint-2000/model.safetensors +3 -0
  20. checkpoint-2000/optimizer.pt +3 -0
  21. checkpoint-2000/rng_state.pth +3 -0
  22. checkpoint-2000/scheduler.pt +3 -0
  23. checkpoint-2000/trainer_state.json +62 -0
  24. checkpoint-2000/training_args.bin +3 -0
  25. checkpoint-2500/config.json +38 -0
  26. checkpoint-2500/generation_config.json +6 -0
  27. checkpoint-2500/model.safetensors +3 -0
  28. checkpoint-2500/optimizer.pt +3 -0
  29. checkpoint-2500/rng_state.pth +3 -0
  30. checkpoint-2500/scheduler.pt +3 -0
  31. checkpoint-2500/trainer_state.json +69 -0
  32. checkpoint-2500/training_args.bin +3 -0
  33. checkpoint-3000/config.json +38 -0
  34. checkpoint-3000/generation_config.json +6 -0
  35. checkpoint-3000/model.safetensors +3 -0
  36. checkpoint-3000/optimizer.pt +3 -0
  37. checkpoint-3000/rng_state.pth +3 -0
  38. checkpoint-3000/scheduler.pt +3 -0
  39. checkpoint-3000/trainer_state.json +76 -0
  40. checkpoint-3000/training_args.bin +3 -0
  41. checkpoint-3500/config.json +38 -0
  42. checkpoint-3500/generation_config.json +6 -0
  43. checkpoint-3500/model.safetensors +3 -0
  44. checkpoint-3500/optimizer.pt +3 -0
  45. checkpoint-3500/rng_state.pth +3 -0
  46. checkpoint-3500/scheduler.pt +3 -0
  47. checkpoint-3500/trainer_state.json +83 -0
  48. checkpoint-3500/training_args.bin +3 -0
  49. checkpoint-4000/config.json +38 -0
  50. checkpoint-4000/generation_config.json +6 -0
checkpoint-1000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 50256,
10
+ "initializer_range": 0.02,
11
+ "layer_norm_epsilon": 1e-05,
12
+ "model_type": "gpt2",
13
+ "n_ctx": 1024,
14
+ "n_embd": 768,
15
+ "n_head": 12,
16
+ "n_inner": null,
17
+ "n_layer": 12,
18
+ "n_positions": 1024,
19
+ "reorder_and_upcast_attn": false,
20
+ "resid_pdrop": 0.1,
21
+ "scale_attn_by_inverse_layer_idx": false,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.50.0",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
checkpoint-1000/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.50.0"
6
+ }
checkpoint-1000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5f29d91c8113d1fccb3de5aa208450ca9977e3e73106b331b5b834c02c968e8
3
+ size 497774208
checkpoint-1000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a85b0f11007e08c390566742a151267e2c29335761b97a4590b9f0cfd21bcb06
3
+ size 995642298
checkpoint-1000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9de967e1901a1f83ee62c42218324ff0ef95543cf28963c868cc8805f356203
3
+ size 14244
checkpoint-1000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5255b856e77ae07b5d665e31bcb5982e777d28e3803bba67e981282ca7f3efc2
3
+ size 1064
checkpoint-1000/trainer_state.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.10893246187363835,
6
+ "eval_steps": 500,
7
+ "global_step": 1000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.054466230936819175,
14
+ "grad_norm": 5.168296813964844,
15
+ "learning_rate": 5e-05,
16
+ "loss": 1.747,
17
+ "step": 500
18
+ },
19
+ {
20
+ "epoch": 0.10893246187363835,
21
+ "grad_norm": 0.6366938948631287,
22
+ "learning_rate": 4.711981566820277e-05,
23
+ "loss": 1.3516,
24
+ "step": 1000
25
+ }
26
+ ],
27
+ "logging_steps": 500,
28
+ "max_steps": 9180,
29
+ "num_input_tokens_seen": 0,
30
+ "num_train_epochs": 1,
31
+ "save_steps": 500,
32
+ "stateful_callbacks": {
33
+ "TrainerControl": {
34
+ "args": {
35
+ "should_epoch_stop": false,
36
+ "should_evaluate": false,
37
+ "should_log": false,
38
+ "should_save": true,
39
+ "should_training_stop": false
40
+ },
41
+ "attributes": {}
42
+ }
43
+ },
44
+ "total_flos": 261292032000000.0,
45
+ "train_batch_size": 4,
46
+ "trial_name": null,
47
+ "trial_params": null
48
+ }
checkpoint-1000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbab60857fc9fddbc7593bfddee8002451e5c50b78b888987fa868bd9cabe9b8
3
+ size 5304
checkpoint-1500/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 50256,
10
+ "initializer_range": 0.02,
11
+ "layer_norm_epsilon": 1e-05,
12
+ "model_type": "gpt2",
13
+ "n_ctx": 1024,
14
+ "n_embd": 768,
15
+ "n_head": 12,
16
+ "n_inner": null,
17
+ "n_layer": 12,
18
+ "n_positions": 1024,
19
+ "reorder_and_upcast_attn": false,
20
+ "resid_pdrop": 0.1,
21
+ "scale_attn_by_inverse_layer_idx": false,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.50.0",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
checkpoint-1500/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.50.0"
6
+ }
checkpoint-1500/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d74bf00b6b6d893e15990c1f046404c7ef6d67a6b925f403e926e17ee52f3d67
3
+ size 497774208
checkpoint-1500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96c11471d1a5066b299b467ba0fb98cb39e8254ee9b156fb8e603fa614931322
3
+ size 995642298
checkpoint-1500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a52b643f4da984a3fc7ecc63f0d369cb90f514ea973ef64d8460181f91561448
3
+ size 14244
checkpoint-1500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d056bf19d2ed63fb9824962abd8fe831ffa71f680bd984206b22c08319014dd3
3
+ size 1064
checkpoint-1500/trainer_state.json ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.16339869281045752,
6
+ "eval_steps": 500,
7
+ "global_step": 1500,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.054466230936819175,
14
+ "grad_norm": 5.168296813964844,
15
+ "learning_rate": 5e-05,
16
+ "loss": 1.747,
17
+ "step": 500
18
+ },
19
+ {
20
+ "epoch": 0.10893246187363835,
21
+ "grad_norm": 0.6366938948631287,
22
+ "learning_rate": 4.711981566820277e-05,
23
+ "loss": 1.3516,
24
+ "step": 1000
25
+ },
26
+ {
27
+ "epoch": 0.16339869281045752,
28
+ "grad_norm": 0.5679114460945129,
29
+ "learning_rate": 4.423963133640553e-05,
30
+ "loss": 1.3809,
31
+ "step": 1500
32
+ }
33
+ ],
34
+ "logging_steps": 500,
35
+ "max_steps": 9180,
36
+ "num_input_tokens_seen": 0,
37
+ "num_train_epochs": 1,
38
+ "save_steps": 500,
39
+ "stateful_callbacks": {
40
+ "TrainerControl": {
41
+ "args": {
42
+ "should_epoch_stop": false,
43
+ "should_evaluate": false,
44
+ "should_log": false,
45
+ "should_save": true,
46
+ "should_training_stop": false
47
+ },
48
+ "attributes": {}
49
+ }
50
+ },
51
+ "total_flos": 391938048000000.0,
52
+ "train_batch_size": 4,
53
+ "trial_name": null,
54
+ "trial_params": null
55
+ }
checkpoint-1500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbab60857fc9fddbc7593bfddee8002451e5c50b78b888987fa868bd9cabe9b8
3
+ size 5304
checkpoint-2000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 50256,
10
+ "initializer_range": 0.02,
11
+ "layer_norm_epsilon": 1e-05,
12
+ "model_type": "gpt2",
13
+ "n_ctx": 1024,
14
+ "n_embd": 768,
15
+ "n_head": 12,
16
+ "n_inner": null,
17
+ "n_layer": 12,
18
+ "n_positions": 1024,
19
+ "reorder_and_upcast_attn": false,
20
+ "resid_pdrop": 0.1,
21
+ "scale_attn_by_inverse_layer_idx": false,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.50.0",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
checkpoint-2000/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.50.0"
6
+ }
checkpoint-2000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9fd5e730b214d372bd72f8c4be30bddff339a7478680626139ab19ca81437248
3
+ size 497774208
checkpoint-2000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d86438c1f87fa6cafaf2248c39e16258c155c9dcc346201d2754a1a27b8fd4d
3
+ size 995642298
checkpoint-2000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21dab0a710a52b38589f7c68fcbe22f2f97319c3fc4c23b47e7f7f450adc99af
3
+ size 14244
checkpoint-2000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e01b325926de808d22fd22f729777aed49c3f3e89abceff3fc429ae98b7ed6a5
3
+ size 1064
checkpoint-2000/trainer_state.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.2178649237472767,
6
+ "eval_steps": 500,
7
+ "global_step": 2000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.054466230936819175,
14
+ "grad_norm": 5.168296813964844,
15
+ "learning_rate": 5e-05,
16
+ "loss": 1.747,
17
+ "step": 500
18
+ },
19
+ {
20
+ "epoch": 0.10893246187363835,
21
+ "grad_norm": 0.6366938948631287,
22
+ "learning_rate": 4.711981566820277e-05,
23
+ "loss": 1.3516,
24
+ "step": 1000
25
+ },
26
+ {
27
+ "epoch": 0.16339869281045752,
28
+ "grad_norm": 0.5679114460945129,
29
+ "learning_rate": 4.423963133640553e-05,
30
+ "loss": 1.3809,
31
+ "step": 1500
32
+ },
33
+ {
34
+ "epoch": 0.2178649237472767,
35
+ "grad_norm": 1.4229077100753784,
36
+ "learning_rate": 4.13594470046083e-05,
37
+ "loss": 1.2784,
38
+ "step": 2000
39
+ }
40
+ ],
41
+ "logging_steps": 500,
42
+ "max_steps": 9180,
43
+ "num_input_tokens_seen": 0,
44
+ "num_train_epochs": 1,
45
+ "save_steps": 500,
46
+ "stateful_callbacks": {
47
+ "TrainerControl": {
48
+ "args": {
49
+ "should_epoch_stop": false,
50
+ "should_evaluate": false,
51
+ "should_log": false,
52
+ "should_save": true,
53
+ "should_training_stop": false
54
+ },
55
+ "attributes": {}
56
+ }
57
+ },
58
+ "total_flos": 522584064000000.0,
59
+ "train_batch_size": 4,
60
+ "trial_name": null,
61
+ "trial_params": null
62
+ }
checkpoint-2000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbab60857fc9fddbc7593bfddee8002451e5c50b78b888987fa868bd9cabe9b8
3
+ size 5304
checkpoint-2500/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 50256,
10
+ "initializer_range": 0.02,
11
+ "layer_norm_epsilon": 1e-05,
12
+ "model_type": "gpt2",
13
+ "n_ctx": 1024,
14
+ "n_embd": 768,
15
+ "n_head": 12,
16
+ "n_inner": null,
17
+ "n_layer": 12,
18
+ "n_positions": 1024,
19
+ "reorder_and_upcast_attn": false,
20
+ "resid_pdrop": 0.1,
21
+ "scale_attn_by_inverse_layer_idx": false,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.50.0",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
checkpoint-2500/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.50.0"
6
+ }
checkpoint-2500/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbe6089db78a6f9849494c806faaac7e44ac06167820943f239629c1da626ed7
3
+ size 497774208
checkpoint-2500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0769540809050ac34b27b1f37217afd4bdbf73775c58507dd43b5d99cef1af0d
3
+ size 995642298
checkpoint-2500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cbc456a1b551ff64da97202cd09b035aa3c7518ddb546dbc0ed234a924f9f9e0
3
+ size 14244
checkpoint-2500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eaa960551ecb6751541543af4e08deb409de73233d127d35292337960fdabc3a
3
+ size 1064
checkpoint-2500/trainer_state.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.27233115468409586,
6
+ "eval_steps": 500,
7
+ "global_step": 2500,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.054466230936819175,
14
+ "grad_norm": 5.168296813964844,
15
+ "learning_rate": 5e-05,
16
+ "loss": 1.747,
17
+ "step": 500
18
+ },
19
+ {
20
+ "epoch": 0.10893246187363835,
21
+ "grad_norm": 0.6366938948631287,
22
+ "learning_rate": 4.711981566820277e-05,
23
+ "loss": 1.3516,
24
+ "step": 1000
25
+ },
26
+ {
27
+ "epoch": 0.16339869281045752,
28
+ "grad_norm": 0.5679114460945129,
29
+ "learning_rate": 4.423963133640553e-05,
30
+ "loss": 1.3809,
31
+ "step": 1500
32
+ },
33
+ {
34
+ "epoch": 0.2178649237472767,
35
+ "grad_norm": 1.4229077100753784,
36
+ "learning_rate": 4.13594470046083e-05,
37
+ "loss": 1.2784,
38
+ "step": 2000
39
+ },
40
+ {
41
+ "epoch": 0.27233115468409586,
42
+ "grad_norm": 1.4274455308914185,
43
+ "learning_rate": 3.847926267281106e-05,
44
+ "loss": 1.2724,
45
+ "step": 2500
46
+ }
47
+ ],
48
+ "logging_steps": 500,
49
+ "max_steps": 9180,
50
+ "num_input_tokens_seen": 0,
51
+ "num_train_epochs": 1,
52
+ "save_steps": 500,
53
+ "stateful_callbacks": {
54
+ "TrainerControl": {
55
+ "args": {
56
+ "should_epoch_stop": false,
57
+ "should_evaluate": false,
58
+ "should_log": false,
59
+ "should_save": true,
60
+ "should_training_stop": false
61
+ },
62
+ "attributes": {}
63
+ }
64
+ },
65
+ "total_flos": 653230080000000.0,
66
+ "train_batch_size": 4,
67
+ "trial_name": null,
68
+ "trial_params": null
69
+ }
checkpoint-2500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbab60857fc9fddbc7593bfddee8002451e5c50b78b888987fa868bd9cabe9b8
3
+ size 5304
checkpoint-3000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 50256,
10
+ "initializer_range": 0.02,
11
+ "layer_norm_epsilon": 1e-05,
12
+ "model_type": "gpt2",
13
+ "n_ctx": 1024,
14
+ "n_embd": 768,
15
+ "n_head": 12,
16
+ "n_inner": null,
17
+ "n_layer": 12,
18
+ "n_positions": 1024,
19
+ "reorder_and_upcast_attn": false,
20
+ "resid_pdrop": 0.1,
21
+ "scale_attn_by_inverse_layer_idx": false,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.50.0",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
checkpoint-3000/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.50.0"
6
+ }
checkpoint-3000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:429179dc830847608cf995a0c189b5fe8432d388d05d5dbbcf1f73eeff8ae722
3
+ size 497774208
checkpoint-3000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac766763cfe2d922875ecc4ca62a2aaa918557cd309b5c17307d9c85445db728
3
+ size 995642298
checkpoint-3000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa1b086467d773850d77f5d1925d8b78501e5b7b4d87ddada3a4275e62b85377
3
+ size 14244
checkpoint-3000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c7701509355468b66df64bc88943f8739c244356a7900b57d3fac69bdbc3a42
3
+ size 1064
checkpoint-3000/trainer_state.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.32679738562091504,
6
+ "eval_steps": 500,
7
+ "global_step": 3000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.054466230936819175,
14
+ "grad_norm": 5.168296813964844,
15
+ "learning_rate": 5e-05,
16
+ "loss": 1.747,
17
+ "step": 500
18
+ },
19
+ {
20
+ "epoch": 0.10893246187363835,
21
+ "grad_norm": 0.6366938948631287,
22
+ "learning_rate": 4.711981566820277e-05,
23
+ "loss": 1.3516,
24
+ "step": 1000
25
+ },
26
+ {
27
+ "epoch": 0.16339869281045752,
28
+ "grad_norm": 0.5679114460945129,
29
+ "learning_rate": 4.423963133640553e-05,
30
+ "loss": 1.3809,
31
+ "step": 1500
32
+ },
33
+ {
34
+ "epoch": 0.2178649237472767,
35
+ "grad_norm": 1.4229077100753784,
36
+ "learning_rate": 4.13594470046083e-05,
37
+ "loss": 1.2784,
38
+ "step": 2000
39
+ },
40
+ {
41
+ "epoch": 0.27233115468409586,
42
+ "grad_norm": 1.4274455308914185,
43
+ "learning_rate": 3.847926267281106e-05,
44
+ "loss": 1.2724,
45
+ "step": 2500
46
+ },
47
+ {
48
+ "epoch": 0.32679738562091504,
49
+ "grad_norm": 1.3789653778076172,
50
+ "learning_rate": 3.559907834101383e-05,
51
+ "loss": 1.2497,
52
+ "step": 3000
53
+ }
54
+ ],
55
+ "logging_steps": 500,
56
+ "max_steps": 9180,
57
+ "num_input_tokens_seen": 0,
58
+ "num_train_epochs": 1,
59
+ "save_steps": 500,
60
+ "stateful_callbacks": {
61
+ "TrainerControl": {
62
+ "args": {
63
+ "should_epoch_stop": false,
64
+ "should_evaluate": false,
65
+ "should_log": false,
66
+ "should_save": true,
67
+ "should_training_stop": false
68
+ },
69
+ "attributes": {}
70
+ }
71
+ },
72
+ "total_flos": 783876096000000.0,
73
+ "train_batch_size": 4,
74
+ "trial_name": null,
75
+ "trial_params": null
76
+ }
checkpoint-3000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbab60857fc9fddbc7593bfddee8002451e5c50b78b888987fa868bd9cabe9b8
3
+ size 5304
checkpoint-3500/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 50256,
10
+ "initializer_range": 0.02,
11
+ "layer_norm_epsilon": 1e-05,
12
+ "model_type": "gpt2",
13
+ "n_ctx": 1024,
14
+ "n_embd": 768,
15
+ "n_head": 12,
16
+ "n_inner": null,
17
+ "n_layer": 12,
18
+ "n_positions": 1024,
19
+ "reorder_and_upcast_attn": false,
20
+ "resid_pdrop": 0.1,
21
+ "scale_attn_by_inverse_layer_idx": false,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.50.0",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
checkpoint-3500/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.50.0"
6
+ }
checkpoint-3500/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:720835076f3e65ea605ce1f4bbf0f0fb79caea39712d572ed547b44e045d3a23
3
+ size 497774208
checkpoint-3500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c806075de53cbc330d879bc1ac17d2bda86f90e2489acf45d970003de31fcba4
3
+ size 995642298
checkpoint-3500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c2b2a7ae2e95906148def4e9d2a8982d4192a6ed553f8db64418d4f9b312e4d
3
+ size 14244
checkpoint-3500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d989adbe7b477a299784449defb3b7e0be2bffc7393bfb9e63c4c3f29a629396
3
+ size 1064
checkpoint-3500/trainer_state.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.3812636165577342,
6
+ "eval_steps": 500,
7
+ "global_step": 3500,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.054466230936819175,
14
+ "grad_norm": 5.168296813964844,
15
+ "learning_rate": 5e-05,
16
+ "loss": 1.747,
17
+ "step": 500
18
+ },
19
+ {
20
+ "epoch": 0.10893246187363835,
21
+ "grad_norm": 0.6366938948631287,
22
+ "learning_rate": 4.711981566820277e-05,
23
+ "loss": 1.3516,
24
+ "step": 1000
25
+ },
26
+ {
27
+ "epoch": 0.16339869281045752,
28
+ "grad_norm": 0.5679114460945129,
29
+ "learning_rate": 4.423963133640553e-05,
30
+ "loss": 1.3809,
31
+ "step": 1500
32
+ },
33
+ {
34
+ "epoch": 0.2178649237472767,
35
+ "grad_norm": 1.4229077100753784,
36
+ "learning_rate": 4.13594470046083e-05,
37
+ "loss": 1.2784,
38
+ "step": 2000
39
+ },
40
+ {
41
+ "epoch": 0.27233115468409586,
42
+ "grad_norm": 1.4274455308914185,
43
+ "learning_rate": 3.847926267281106e-05,
44
+ "loss": 1.2724,
45
+ "step": 2500
46
+ },
47
+ {
48
+ "epoch": 0.32679738562091504,
49
+ "grad_norm": 1.3789653778076172,
50
+ "learning_rate": 3.559907834101383e-05,
51
+ "loss": 1.2497,
52
+ "step": 3000
53
+ },
54
+ {
55
+ "epoch": 0.3812636165577342,
56
+ "grad_norm": 1.8324720859527588,
57
+ "learning_rate": 3.271889400921659e-05,
58
+ "loss": 1.2794,
59
+ "step": 3500
60
+ }
61
+ ],
62
+ "logging_steps": 500,
63
+ "max_steps": 9180,
64
+ "num_input_tokens_seen": 0,
65
+ "num_train_epochs": 1,
66
+ "save_steps": 500,
67
+ "stateful_callbacks": {
68
+ "TrainerControl": {
69
+ "args": {
70
+ "should_epoch_stop": false,
71
+ "should_evaluate": false,
72
+ "should_log": false,
73
+ "should_save": true,
74
+ "should_training_stop": false
75
+ },
76
+ "attributes": {}
77
+ }
78
+ },
79
+ "total_flos": 914522112000000.0,
80
+ "train_batch_size": 4,
81
+ "trial_name": null,
82
+ "trial_params": null
83
+ }
checkpoint-3500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbab60857fc9fddbc7593bfddee8002451e5c50b78b888987fa868bd9cabe9b8
3
+ size 5304
checkpoint-4000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_function": "gelu_new",
3
+ "architectures": [
4
+ "GPT2LMHeadModel"
5
+ ],
6
+ "attn_pdrop": 0.1,
7
+ "bos_token_id": 50256,
8
+ "embd_pdrop": 0.1,
9
+ "eos_token_id": 50256,
10
+ "initializer_range": 0.02,
11
+ "layer_norm_epsilon": 1e-05,
12
+ "model_type": "gpt2",
13
+ "n_ctx": 1024,
14
+ "n_embd": 768,
15
+ "n_head": 12,
16
+ "n_inner": null,
17
+ "n_layer": 12,
18
+ "n_positions": 1024,
19
+ "reorder_and_upcast_attn": false,
20
+ "resid_pdrop": 0.1,
21
+ "scale_attn_by_inverse_layer_idx": false,
22
+ "scale_attn_weights": true,
23
+ "summary_activation": null,
24
+ "summary_first_dropout": 0.1,
25
+ "summary_proj_to_labels": true,
26
+ "summary_type": "cls_index",
27
+ "summary_use_proj": true,
28
+ "task_specific_params": {
29
+ "text-generation": {
30
+ "do_sample": true,
31
+ "max_length": 50
32
+ }
33
+ },
34
+ "torch_dtype": "float32",
35
+ "transformers_version": "4.50.0",
36
+ "use_cache": true,
37
+ "vocab_size": 50257
38
+ }
checkpoint-4000/generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.50.0"
6
+ }