Alex-xu commited on
Commit
cad04c7
1 Parent(s): c7deebc

Training in progress, step 9500, checkpoint

Browse files
checkpoint-9500/config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LongelmForMaskedLMWithEdgePrediction"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "block_size": 8,
7
+ "bos_token_id": 1,
8
+ "classifier_dropout": null,
9
+ "eos_token_id": 2,
10
+ "ep_add_linear_projection": true,
11
+ "global_memory_size": 1,
12
+ "hidden_act": "gelu",
13
+ "hidden_dropout_prob": 0.1,
14
+ "hidden_size": 768,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 3072,
17
+ "layer_norm_eps": 1e-12,
18
+ "max_blocks": 200,
19
+ "max_position_embeddings": 1805,
20
+ "max_relative_position_embeddings": 8,
21
+ "model_type": "longelm",
22
+ "node_size": 1,
23
+ "num_attention_heads": 12,
24
+ "num_hidden_layers": 12,
25
+ "pad_token_id": 3,
26
+ "position_embedding_type": "mixed",
27
+ "torch_dtype": "float32",
28
+ "transformers_version": "4.34.1",
29
+ "type_vocab_size": 2,
30
+ "use_cache": true,
31
+ "vocab_size": 50265
32
+ }
checkpoint-9500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52521da93974bf178a354500a7d78cacd99f65bed0bddc742dd77ee8393b4e8a
3
+ size 1010431994
checkpoint-9500/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5122193fceea0de1677bd7a20ca99c118d16e21dc9db81aa021c1595ba2bb1bf
3
+ size 505200682
checkpoint-9500/rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4bfc0e9e4141bc8e5305367e8be21fb639553795bc45eabdff0f866714f3e85a
3
+ size 15984
checkpoint-9500/rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc4129f51e32dafa979bd271b958287b06d5ecce42f4f61b235a0a8d601626e1
3
+ size 15984
checkpoint-9500/rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e66929460dc8024425d40511d37bd1acfe69ca517832d12490a5fa9872f59d7
3
+ size 15984
checkpoint-9500/rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a35dc7eb963e606f2b4939a8c7a6a3fe12addd1dd60945913656efec798cd6b6
3
+ size 15984
checkpoint-9500/rng_state_4.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d72df70faf47cc6cf01450558dfda071b6687f761113cb742d30a6862f0290c
3
+ size 15984
checkpoint-9500/rng_state_5.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a718728a1558101129cf6a5dec946144667a39f88dbeafa193d051c0a062aa86
3
+ size 15984
checkpoint-9500/rng_state_6.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9ee3e68478f3accd4dbc269ba575a2f4e841c801bc0a68333b751a5e1b31ed1
3
+ size 15984
checkpoint-9500/rng_state_7.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:774a72e1651fbb8c3d13d27da7c1046c5dd37afd1a9441ec30de9c0504130e6a
3
+ size 15984
checkpoint-9500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d59cd4e6e0a781b313e6063f49eb29dc80fc8e1748abc14b5492d23b3818c02
3
+ size 1064
checkpoint-9500/special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": {
3
+ "content": "[CLS]",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "mask_token": {
10
+ "content": "[MASK]",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "[PAD]",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "sep_token": {
24
+ "content": "[SEP]",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "[UNK]",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
checkpoint-9500/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-9500/tokenizer_config.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[UNK]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "[CLS]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "[SEP]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "[PAD]",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "[MASK]",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "30522": {
44
+ "content": "<INST>",
45
+ "lstrip": false,
46
+ "normalized": true,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": false
50
+ }
51
+ },
52
+ "clean_up_tokenization_spaces": true,
53
+ "cls_token": "[CLS]",
54
+ "mask_token": "[MASK]",
55
+ "model_max_length": 512,
56
+ "pad_token": "[PAD]",
57
+ "sep_token": "[SEP]",
58
+ "tokenizer_class": "LongelmTokenizer",
59
+ "unk_token": "[UNK]"
60
+ }
checkpoint-9500/trainer_state.json ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.7303829705444236,
5
+ "eval_steps": 500,
6
+ "global_step": 9500,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "ep_loss": 4.4238,
13
+ "epoch": 0.04,
14
+ "learning_rate": 2.5e-05,
15
+ "loss": 11.1467,
16
+ "mlm_loss": 6.7229,
17
+ "step": 500
18
+ },
19
+ {
20
+ "ep_loss": 0.4305,
21
+ "epoch": 0.08,
22
+ "learning_rate": 5e-05,
23
+ "loss": 2.9841,
24
+ "mlm_loss": 2.5537,
25
+ "step": 1000
26
+ },
27
+ {
28
+ "ep_loss": 0.3724,
29
+ "epoch": 0.12,
30
+ "learning_rate": 7.5e-05,
31
+ "loss": 2.2371,
32
+ "mlm_loss": 1.8647,
33
+ "step": 1500
34
+ },
35
+ {
36
+ "ep_loss": 0.364,
37
+ "epoch": 0.15,
38
+ "learning_rate": 0.0001,
39
+ "loss": 1.8477,
40
+ "mlm_loss": 1.4837,
41
+ "step": 2000
42
+ },
43
+ {
44
+ "ep_loss": 0.3678,
45
+ "epoch": 0.19,
46
+ "learning_rate": 0.00012495,
47
+ "loss": 1.5215,
48
+ "mlm_loss": 1.1538,
49
+ "step": 2500
50
+ },
51
+ {
52
+ "ep_loss": 0.3617,
53
+ "epoch": 0.23,
54
+ "learning_rate": 0.00014995,
55
+ "loss": 1.4119,
56
+ "mlm_loss": 1.0501,
57
+ "step": 3000
58
+ },
59
+ {
60
+ "ep_loss": 0.3336,
61
+ "epoch": 0.27,
62
+ "learning_rate": 0.0001749,
63
+ "loss": 1.3027,
64
+ "mlm_loss": 0.9691,
65
+ "step": 3500
66
+ },
67
+ {
68
+ "ep_loss": 0.3348,
69
+ "epoch": 0.31,
70
+ "learning_rate": 0.0001999,
71
+ "loss": 1.2441,
72
+ "mlm_loss": 0.9093,
73
+ "step": 4000
74
+ },
75
+ {
76
+ "ep_loss": 0.3348,
77
+ "epoch": 0.35,
78
+ "learning_rate": 0.0002249,
79
+ "loss": 1.1942,
80
+ "mlm_loss": 0.8594,
81
+ "step": 4500
82
+ },
83
+ {
84
+ "ep_loss": 0.3331,
85
+ "epoch": 0.38,
86
+ "learning_rate": 0.0002499,
87
+ "loss": 1.1466,
88
+ "mlm_loss": 0.8135,
89
+ "step": 5000
90
+ },
91
+ {
92
+ "ep_loss": 0.3268,
93
+ "epoch": 0.42,
94
+ "learning_rate": 0.00027489999999999996,
95
+ "loss": 1.1067,
96
+ "mlm_loss": 0.7799,
97
+ "step": 5500
98
+ },
99
+ {
100
+ "ep_loss": 0.3378,
101
+ "epoch": 0.46,
102
+ "learning_rate": 0.00029985,
103
+ "loss": 1.1007,
104
+ "mlm_loss": 0.7629,
105
+ "step": 6000
106
+ },
107
+ {
108
+ "ep_loss": 0.3033,
109
+ "epoch": 0.5,
110
+ "learning_rate": 0.00032485,
111
+ "loss": 1.028,
112
+ "mlm_loss": 0.7246,
113
+ "step": 6500
114
+ },
115
+ {
116
+ "ep_loss": 0.2848,
117
+ "epoch": 0.54,
118
+ "learning_rate": 0.00034985,
119
+ "loss": 0.9833,
120
+ "mlm_loss": 0.6985,
121
+ "step": 7000
122
+ },
123
+ {
124
+ "ep_loss": 0.316,
125
+ "epoch": 0.58,
126
+ "learning_rate": 0.00037485000000000003,
127
+ "loss": 1.0235,
128
+ "mlm_loss": 0.7076,
129
+ "step": 7500
130
+ },
131
+ {
132
+ "ep_loss": 0.2885,
133
+ "epoch": 0.62,
134
+ "learning_rate": 0.00039975,
135
+ "loss": 0.9673,
136
+ "mlm_loss": 0.6789,
137
+ "step": 8000
138
+ },
139
+ {
140
+ "ep_loss": 0.274,
141
+ "epoch": 0.65,
142
+ "learning_rate": 0.00042475000000000005,
143
+ "loss": 0.9266,
144
+ "mlm_loss": 0.6525,
145
+ "step": 8500
146
+ },
147
+ {
148
+ "ep_loss": 0.3008,
149
+ "epoch": 0.69,
150
+ "learning_rate": 0.0004497,
151
+ "loss": 0.9646,
152
+ "mlm_loss": 0.6638,
153
+ "step": 9000
154
+ },
155
+ {
156
+ "ep_loss": 0.2843,
157
+ "epoch": 0.73,
158
+ "learning_rate": 0.00047470000000000005,
159
+ "loss": 0.9349,
160
+ "mlm_loss": 0.6506,
161
+ "step": 9500
162
+ }
163
+ ],
164
+ "logging_steps": 500,
165
+ "max_steps": 520240,
166
+ "num_train_epochs": 40,
167
+ "save_steps": 500,
168
+ "total_flos": 9.070764083527025e+18,
169
+ "trial_name": null,
170
+ "trial_params": null
171
+ }
checkpoint-9500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b26e03be70ebe5d40a81a81be94421c0578a9d8742b065bc029501df48632957
3
+ size 4664