Step... (9000/50000 | Loss: 1.7518370151519775, Acc: 0.6520029902458191): 18%|█████▎ | 9175/50000 [3:31:42<14:26:35, 1.27s/it]

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b54768633b7c94c65dc0adcd52791094c373de64dec9c2313b790651064030f8
+oid sha256:55484b434d505ef7284a42471c8326f9bebe13561d6cbe478c61990f9fd7a04d
 size 249750019

outputs/checkpoints/checkpoint-2000/training_state.json

@@ -1 +0,0 @@
-{"step": 2001}

outputs/checkpoints/checkpoint-3000/training_state.json

@@ -1 +0,0 @@
-{"step": 3001}

outputs/checkpoints/checkpoint-4000/training_state.json

@@ -1 +0,0 @@
-{"step": 4001}

outputs/checkpoints/{checkpoint-2000 → checkpoint-7000}/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c1f1112c5a4f38297c063f72b4595e44da861d8a37002bfef8f6a7b8a2db074
+oid sha256:353e62a7bbf3b5817b869c37e749c8e30fe14477d32a3cf95345a030057ed760
 size 249750019

outputs/checkpoints/{checkpoint-4000 → checkpoint-7000}/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e7ff0681be4448d2a38ba748fadb3bb0e2972be224603c18f86f5c8d4f003cd
+oid sha256:0cd67c6ccf30e42fa238a68d1aa1ae063e8e11fc6c50bf034163444ab3f91118
 size 499500278

outputs/checkpoints/checkpoint-7000/training_state.json

@@ -0,0 +1 @@
+{"step": 7001}

outputs/checkpoints/{checkpoint-4000 → checkpoint-8000}/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4feb05268279af0817ec9cdbce949fd0a1aedac0c5d72c727a4bddd1667016a7
+oid sha256:fcd1e001a114c411bab4cde0ffdf4e4bc13e918b2c1c3cac7a75100e5a3f0349
 size 249750019

outputs/checkpoints/{checkpoint-2000 → checkpoint-8000}/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5336f60347962af6217ec50bb816aa2b7a3225125f16bc4da276394b3b7eac9e
+oid sha256:7070a9b0eb3c596cc8b7f538faa458611e2d751b69600e272ec31b7c5c1bbc82
 size 499500278

outputs/checkpoints/checkpoint-8000/training_state.json

@@ -0,0 +1 @@
+{"step": 8001}

outputs/checkpoints/{checkpoint-3000 → checkpoint-9000}/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6c49ca76d9eb4fcaedf18790304282801372115ea7b8926caed70d3f347c9eb
+oid sha256:55484b434d505ef7284a42471c8326f9bebe13561d6cbe478c61990f9fd7a04d
 size 249750019

outputs/checkpoints/{checkpoint-3000 → checkpoint-9000}/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c4b4dd23e9ef7da6242bfe8be61c752f257574633b2f31de4d3a3612ea946453
+oid sha256:2085e2cdeca180d85963536b92e396dad244a1a40804023af28d868e886658c8
 size 499500278

outputs/checkpoints/checkpoint-9000/training_state.json

@@ -0,0 +1 @@
+{"step": 9001}

outputs/events.out.tfevents.1627258355.tablespoon.3000110.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f97666571180797f8a968d48c3763a8cbeb077e610e79e3950db38d6c6a96a2f
-size 957164
+oid sha256:59ffcc0c842038889bc43ba9ce06f442be97edfc66757b41c7b3292ee06bd1b0
+size 1325429

outputs/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b54768633b7c94c65dc0adcd52791094c373de64dec9c2313b790651064030f8
+oid sha256:55484b434d505ef7284a42471c8326f9bebe13561d6cbe478c61990f9fd7a04d
 size 249750019

outputs/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db32f6c60858ce8df9fec4b0aab38feff295deb2af84540eced3d8fc2c9e95bc
+oid sha256:2085e2cdeca180d85963536b92e396dad244a1a40804023af28d868e886658c8
 size 499500278

outputs/training_state.json

@@ -1 +1 @@
-{"step": 6001}
+{"step": 9001}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:576a3a8b19cb7a56e505fbb15f9773d81f46cc98eef8577312b45d8af16f6155
+oid sha256:5319ca4762633df47ea8467b2f87c5c43f499ace80a7f0bc7dd075f31d1405fd
 size 498858859

wandb/run-20210726_001233-17u6inbn/files/output.log

@@ -4268,6 +4268,1733 @@ You should probably TRAIN this model on a down-stream task to be able to use it
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (6000/50000 | Loss: 1.7780379056930542, Acc: 0.6486639976501465):  14%|████                         | 7000/50000 [2:40:57<15:48:42,  1.32s/it]
+Evaluating ...:   0%|                                                                                                         | 0/130 [00:00<?, ?it/s]
+Step... (6500 | Loss: 1.835520625114441, Learning Rate: 0.0005272727576084435)
+
+
+
+
+
+
+
+
+
+
+
+
+[04:49:21] - INFO - __main__ - Saving checkpoint at 7000 steps██████████████████████████████████████████████████████| 130/130 [00:21<00:00,  4.59it/s]
+All Flax model weights were used when initializing RobertaForMaskedLM.
+Some weights of RobertaForMaskedLM were not initialized from the Flax model and are newly initialized: ['lm_head.decoder.weight', 'roberta.embeddings.position_ids', 'lm_head.decoder.bias']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (7000/50000 | Loss: 1.767648458480835, Acc: 0.6495990753173828):  16%|████▊                         | 8000/50000 [3:04:03<15:18:38,  1.31s/it]
+Step... (7500 | Loss: 1.8483006954193115, Learning Rate: 0.0005151515360921621)
+Step... (7000/50000 | Loss: 1.767648458480835, Acc: 0.6495990753173828):  16%|████▊                         | 8000/50000 [3:04:04<15:18:38,  1.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+[05:12:28] - INFO - __main__ - Saving checkpoint at 8000 steps██████████████████████████████████████████████████████| 130/130 [00:21<00:00,  4.59it/s]
+All Flax model weights were used when initializing RobertaForMaskedLM.
+Some weights of RobertaForMaskedLM were not initialized from the Flax model and are newly initialized: ['lm_head.decoder.weight', 'roberta.embeddings.position_ids', 'lm_head.decoder.bias']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step... (8000/50000 | Loss: 1.7662373781204224, Acc: 0.6503182649612427):  18%|█████▏                       | 9000/50000 [3:26:55<14:07:58,  1.24s/it]
+Step... (8500 | Loss: 1.8929920196533203, Learning Rate: 0.0005030303145758808)
+Step... (9000 | Loss: 1.841712236404419, Learning Rate: 0.0004969697329215705)
+
+
+
+
+
+
+
+
+
+
+
+[05:35:18] - INFO - __main__ - Saving checkpoint at 9000 steps██████████████████████████████████████████████████████| 130/130 [00:21<00:00,  4.60it/s]
+All Flax model weights were used when initializing RobertaForMaskedLM.
+Some weights of RobertaForMaskedLM were not initialized from the Flax model and are newly initialized: ['lm_head.decoder.weight', 'roberta.embeddings.position_ids', 'lm_head.decoder.bias']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 

wandb/run-20210726_001233-17u6inbn/files/wandb-summary.json

@@ -1 +1 @@
-{"global_step": 6500, "_timestamp": 1627274252.003003, "train_time": 152312.671875, "train_learning_rate": 0.0005272727576084435, "_step": 12961, "train_loss": 1.897655963897705, "eval_accuracy": 0.6486639976501465, "eval_loss": 1.7780379056930542}
+{"global_step": 9000, "_timestamp": 1627277689.819425, "train_time": 242086.03125, "train_learning_rate": 0.0004969697329215705, "_step": 17946, "train_loss": 1.800872564315796, "eval_accuracy": 0.6503182649612427, "eval_loss": 1.7662373781204224}