Training in progress, step 6500, checkpoint

checkpoint-6500/config.json

@@ -16,7 +16,7 @@
   "intermediate_size": 3072,
   "layer_norm_eps": 1e-12,
   "max_blocks": 200,
-  "max_position_embeddings": 1801,
+  "max_position_embeddings": 1805,
   "max_relative_position_embeddings": 8,
   "model_type": "longelm",
   "node_size": 1,

checkpoint-6500/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cafd9385532b5445162239e58b8a5e0127213f1e220c822154583bba81c1587c
-size 1010407418
+oid sha256:78a1bcf829ca59fd5516a454b6a5da8e3254f3c1499d6c3cd3f5471c7183307c
+size 1010431994

checkpoint-6500/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc3c57fdc86b39e9ee3e40c071c9e9d037ef2d31d38dd2883d61f28f6428a8f2
-size 505188394
+oid sha256:b890105b5f6ecc2adfd62a87ff6f06cb65a5ed44a32eb0a059a76a94a9c8f1f7
+size 505200682

checkpoint-6500/trainer_state.json

@@ -9,107 +9,107 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "ep_loss": 4.7343,
+      "ep_loss": 4.4238,
       "epoch": 0.04,
       "learning_rate": 2.5e-05,
-      "loss": 11.6218,
-      "mlm_loss": 6.8875,
+      "loss": 11.1467,
+      "mlm_loss": 6.7229,
       "step": 500
     },
     {
-      "ep_loss": 0.4315,
+      "ep_loss": 0.4305,
       "epoch": 0.08,
       "learning_rate": 5e-05,
-      "loss": 3.0285,
-      "mlm_loss": 2.5969,
+      "loss": 2.9841,
+      "mlm_loss": 2.5537,
       "step": 1000
     },
     {
-      "ep_loss": 0.3811,
+      "ep_loss": 0.3724,
       "epoch": 0.12,
       "learning_rate": 7.5e-05,
-      "loss": 2.2571,
-      "mlm_loss": 1.8761,
+      "loss": 2.2371,
+      "mlm_loss": 1.8647,
       "step": 1500
     },
     {
-      "ep_loss": 0.37,
+      "ep_loss": 0.364,
       "epoch": 0.15,
       "learning_rate": 0.0001,
-      "loss": 1.9695,
-      "mlm_loss": 1.5994,
+      "loss": 1.8477,
+      "mlm_loss": 1.4837,
       "step": 2000
     },
     {
-      "ep_loss": 0.3813,
+      "ep_loss": 0.3678,
       "epoch": 0.19,
       "learning_rate": 0.00012495,
-      "loss": 1.5597,
-      "mlm_loss": 1.1785,
+      "loss": 1.5215,
+      "mlm_loss": 1.1538,
       "step": 2500
     },
     {
-      "ep_loss": 0.3624,
+      "ep_loss": 0.3617,
       "epoch": 0.23,
       "learning_rate": 0.00014995,
-      "loss": 1.4228,
-      "mlm_loss": 1.0604,
+      "loss": 1.4119,
+      "mlm_loss": 1.0501,
       "step": 3000
     },
     {
-      "ep_loss": 0.338,
+      "ep_loss": 0.3336,
       "epoch": 0.27,
-      "learning_rate": 0.00017495,
-      "loss": 1.3108,
-      "mlm_loss": 0.9729,
+      "learning_rate": 0.0001749,
+      "loss": 1.3027,
+      "mlm_loss": 0.9691,
       "step": 3500
     },
     {
-      "ep_loss": 0.3524,
+      "ep_loss": 0.3348,
       "epoch": 0.31,
-      "learning_rate": 0.00019994999999999998,
-      "loss": 1.2734,
-      "mlm_loss": 0.921,
+      "learning_rate": 0.0001999,
+      "loss": 1.2441,
+      "mlm_loss": 0.9093,
       "step": 4000
     },
     {
-      "ep_loss": 0.3423,
+      "ep_loss": 0.3348,
       "epoch": 0.35,
-      "learning_rate": 0.00022495000000000002,
-      "loss": 1.217,
-      "mlm_loss": 0.8747,
+      "learning_rate": 0.0002249,
+      "loss": 1.1942,
+      "mlm_loss": 0.8594,
       "step": 4500
     },
     {
-      "ep_loss": 0.3229,
+      "ep_loss": 0.3331,
       "epoch": 0.38,
       "learning_rate": 0.0002499,
-      "loss": 1.1373,
-      "mlm_loss": 0.8144,
+      "loss": 1.1466,
+      "mlm_loss": 0.8135,
       "step": 5000
     },
     {
-      "ep_loss": 0.3326,
+      "ep_loss": 0.3268,
       "epoch": 0.42,
       "learning_rate": 0.00027489999999999996,
-      "loss": 1.117,
-      "mlm_loss": 0.7844,
+      "loss": 1.1067,
+      "mlm_loss": 0.7799,
       "step": 5500
     },
     {
-      "ep_loss": 0.332,
+      "ep_loss": 0.3378,
       "epoch": 0.46,
       "learning_rate": 0.00029985,
-      "loss": 1.0958,
-      "mlm_loss": 0.7638,
+      "loss": 1.1007,
+      "mlm_loss": 0.7629,
       "step": 6000
     },
     {
-      "ep_loss": 0.3236,
+      "ep_loss": 0.3033,
       "epoch": 0.5,
       "learning_rate": 0.00032485,
-      "loss": 1.0729,
-      "mlm_loss": 0.7493,
+      "loss": 1.028,
+      "mlm_loss": 0.7246,
       "step": 6500
     }
   ],

checkpoint-6500/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cee56d4f38339eac5ecf098381d5222b3783455863547a17ac0df67c8caa7d15
+oid sha256:b26e03be70ebe5d40a81a81be94421c0578a9d8742b065bc029501df48632957
 size 4664