Training in progress, epoch 0

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 12.631578947368421,
-    "total_flos": 2.993942051666657e+17,
-    "train_loss": 0.30815972487131754,
-    "train_runtime": 414.0498,
-    "train_samples_per_second": 10.868,
-    "train_steps_per_second": 0.145
+    "epoch": 16.842105263157894,
+    "total_flos": 3.991922735555543e+17,
+    "train_loss": 0.10449823420494794,
+    "train_runtime": 538.8903,
+    "train_samples_per_second": 11.134,
+    "train_steps_per_second": 0.148
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5971bd635e0f39449daf9128e9f2e74b05a7e70538103ee3f14c792a7b694072
+oid sha256:82be55b5e1454300ec311ab775250164f57e3d94609967a9f507a2d42974c55d
 size 349854120

runs/Jun26_07-59-13_a4870aece808/events.out.tfevents.1719388765.a4870aece808.34.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:574d9148b75910a28a4ac9de4f36941e22e5acb38fae51b9926ddec2be0194a3
+size 4740

runs/Jun26_08-01-05_a4870aece808/events.out.tfevents.1719388875.a4870aece808.34.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d393821bd71b5f027d69c7188830ba628e2ded8cb29f2511fff453534278b9f4
+size 4740

runs/Jun26_08-03-10_a4870aece808/events.out.tfevents.1719389000.a4870aece808.34.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23e9e0471793e8b05b61a74065d36d78ea12963ca0fe6a6700af94b5aabff5c6
+size 5057

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 12.631578947368421,
-    "total_flos": 2.993942051666657e+17,
-    "train_loss": 0.30815972487131754,
-    "train_runtime": 414.0498,
-    "train_samples_per_second": 10.868,
-    "train_steps_per_second": 0.145
+    "epoch": 16.842105263157894,
+    "total_flos": 3.991922735555543e+17,
+    "train_loss": 0.10449823420494794,
+    "train_runtime": 538.8903,
+    "train_samples_per_second": 11.134,
+    "train_steps_per_second": 0.148
 }

trainer_state.json

@@ -1,186 +1,236 @@
 {
   "best_metric": 0.9857142857142858,
-  "best_model_checkpoint": "document-spoof-clip/checkpoint-57",
-  "epoch": 12.631578947368421,
+  "best_model_checkpoint": "document-spoof-clip/checkpoint-9",
+  "epoch": 16.842105263157894,
   "eval_steps": 500,
-  "global_step": 60,
+  "global_step": 80,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.8421052631578947,
-      "eval_accuracy": 0.8285714285714286,
-      "eval_loss": 0.5341095328330994,
-      "eval_runtime": 6.1365,
-      "eval_samples_per_second": 11.407,
-      "eval_steps_per_second": 0.815,
+      "eval_accuracy": 0.9571428571428572,
+      "eval_loss": 0.2112182229757309,
+      "eval_runtime": 5.2092,
+      "eval_samples_per_second": 13.438,
+      "eval_steps_per_second": 0.96,
       "step": 4
     },
     {
       "epoch": 1.8947368421052633,
-      "eval_accuracy": 0.8857142857142857,
-      "eval_loss": 0.2827140986919403,
-      "eval_runtime": 5.2456,
-      "eval_samples_per_second": 13.345,
-      "eval_steps_per_second": 0.953,
+      "eval_accuracy": 0.9857142857142858,
+      "eval_loss": 0.12266874313354492,
+      "eval_runtime": 5.2038,
+      "eval_samples_per_second": 13.452,
+      "eval_steps_per_second": 0.961,
       "step": 9
     },
     {
       "epoch": 2.1052631578947367,
-      "grad_norm": 14.946775436401367,
-      "learning_rate": 4.62962962962963e-05,
-      "loss": 0.6045,
+      "grad_norm": 4.123354434967041,
+      "learning_rate": 4.8611111111111115e-05,
+      "loss": 0.295,
       "step": 10
     },
     {
       "epoch": 2.9473684210526314,
-      "eval_accuracy": 0.7857142857142857,
-      "eval_loss": 0.4043836295604706,
-      "eval_runtime": 5.2087,
-      "eval_samples_per_second": 13.439,
-      "eval_steps_per_second": 0.96,
+      "eval_accuracy": 0.9571428571428572,
+      "eval_loss": 0.12026520818471909,
+      "eval_runtime": 5.3187,
+      "eval_samples_per_second": 13.161,
+      "eval_steps_per_second": 0.94,
       "step": 14
     },
     {
       "epoch": 4.0,
-      "eval_accuracy": 0.9142857142857143,
-      "eval_loss": 0.22778411209583282,
-      "eval_runtime": 5.214,
-      "eval_samples_per_second": 13.425,
-      "eval_steps_per_second": 0.959,
+      "eval_accuracy": 0.9714285714285714,
+      "eval_loss": 0.06349408626556396,
+      "eval_runtime": 5.1916,
+      "eval_samples_per_second": 13.483,
+      "eval_steps_per_second": 0.963,
       "step": 19
     },
     {
       "epoch": 4.2105263157894735,
-      "grad_norm": 11.259857177734375,
-      "learning_rate": 3.7037037037037037e-05,
-      "loss": 0.7684,
+      "grad_norm": 14.247283935546875,
+      "learning_rate": 4.166666666666667e-05,
+      "loss": 0.0962,
       "step": 20
     },
     {
       "epoch": 4.842105263157895,
-      "eval_accuracy": 0.9571428571428572,
-      "eval_loss": 0.1032894104719162,
-      "eval_runtime": 5.2447,
-      "eval_samples_per_second": 13.347,
-      "eval_steps_per_second": 0.953,
+      "eval_accuracy": 0.9428571428571428,
+      "eval_loss": 0.2938525080680847,
+      "eval_runtime": 5.3233,
+      "eval_samples_per_second": 13.15,
+      "eval_steps_per_second": 0.939,
       "step": 23
     },
     {
       "epoch": 5.894736842105263,
-      "eval_accuracy": 0.9714285714285714,
-      "eval_loss": 0.1465865820646286,
-      "eval_runtime": 5.2649,
-      "eval_samples_per_second": 13.295,
-      "eval_steps_per_second": 0.95,
+      "eval_accuracy": 0.9285714285714286,
+      "eval_loss": 0.24831673502922058,
+      "eval_runtime": 5.2672,
+      "eval_samples_per_second": 13.29,
+      "eval_steps_per_second": 0.949,
       "step": 28
     },
     {
       "epoch": 6.315789473684211,
-      "grad_norm": 8.416402816772461,
-      "learning_rate": 2.777777777777778e-05,
-      "loss": 0.2323,
+      "grad_norm": 36.67098617553711,
+      "learning_rate": 3.472222222222222e-05,
+      "loss": 0.163,
       "step": 30
     },
     {
       "epoch": 6.947368421052632,
-      "eval_accuracy": 0.9142857142857143,
-      "eval_loss": 0.22525207698345184,
-      "eval_runtime": 5.3071,
-      "eval_samples_per_second": 13.19,
-      "eval_steps_per_second": 0.942,
+      "eval_accuracy": 0.9857142857142858,
+      "eval_loss": 0.07119516283273697,
+      "eval_runtime": 5.2492,
+      "eval_samples_per_second": 13.335,
+      "eval_steps_per_second": 0.953,
       "step": 33
     },
     {
       "epoch": 8.0,
       "eval_accuracy": 0.9714285714285714,
-      "eval_loss": 0.12320270389318466,
-      "eval_runtime": 5.3285,
-      "eval_samples_per_second": 13.137,
-      "eval_steps_per_second": 0.938,
+      "eval_loss": 0.04743731766939163,
+      "eval_runtime": 5.2254,
+      "eval_samples_per_second": 13.396,
+      "eval_steps_per_second": 0.957,
       "step": 38
     },
     {
       "epoch": 8.421052631578947,
-      "grad_norm": 32.6998176574707,
-      "learning_rate": 1.8518518518518518e-05,
-      "loss": 0.0933,
+      "grad_norm": 14.154749870300293,
+      "learning_rate": 2.777777777777778e-05,
+      "loss": 0.0646,
       "step": 40
     },
     {
       "epoch": 8.842105263157894,
-      "eval_accuracy": 0.9714285714285714,
-      "eval_loss": 0.05824749916791916,
-      "eval_runtime": 5.376,
-      "eval_samples_per_second": 13.021,
-      "eval_steps_per_second": 0.93,
+      "eval_accuracy": 0.9428571428571428,
+      "eval_loss": 0.20123328268527985,
+      "eval_runtime": 5.2671,
+      "eval_samples_per_second": 13.29,
+      "eval_steps_per_second": 0.949,
       "step": 42
     },
     {
       "epoch": 9.894736842105264,
-      "eval_accuracy": 0.9714285714285714,
-      "eval_loss": 0.06094963848590851,
-      "eval_runtime": 5.3183,
-      "eval_samples_per_second": 13.162,
-      "eval_steps_per_second": 0.94,
+      "eval_accuracy": 0.9,
+      "eval_loss": 0.3587024211883545,
+      "eval_runtime": 5.2483,
+      "eval_samples_per_second": 13.338,
+      "eval_steps_per_second": 0.953,
       "step": 47
     },
     {
       "epoch": 10.526315789473685,
-      "grad_norm": 7.5993194580078125,
-      "learning_rate": 9.259259259259259e-06,
-      "loss": 0.067,
+      "grad_norm": 3.9370455741882324,
+      "learning_rate": 2.0833333333333336e-05,
+      "loss": 0.1048,
       "step": 50
     },
     {
       "epoch": 10.947368421052632,
-      "eval_accuracy": 0.9714285714285714,
-      "eval_loss": 0.04527128115296364,
-      "eval_runtime": 5.3452,
-      "eval_samples_per_second": 13.096,
-      "eval_steps_per_second": 0.935,
+      "eval_accuracy": 0.9857142857142858,
+      "eval_loss": 0.04267547279596329,
+      "eval_runtime": 5.2718,
+      "eval_samples_per_second": 13.278,
+      "eval_steps_per_second": 0.948,
       "step": 52
     },
     {
       "epoch": 12.0,
       "eval_accuracy": 0.9857142857142858,
-      "eval_loss": 0.04218948259949684,
-      "eval_runtime": 5.3043,
-      "eval_samples_per_second": 13.197,
-      "eval_steps_per_second": 0.943,
+      "eval_loss": 0.01490915659815073,
+      "eval_runtime": 5.2034,
+      "eval_samples_per_second": 13.453,
+      "eval_steps_per_second": 0.961,
       "step": 57
     },
     {
       "epoch": 12.631578947368421,
-      "grad_norm": 6.4861040115356445,
-      "learning_rate": 0.0,
-      "loss": 0.0835,
+      "grad_norm": 5.009355545043945,
+      "learning_rate": 1.388888888888889e-05,
+      "loss": 0.0519,
       "step": 60
     },
     {
-      "epoch": 12.631578947368421,
-      "eval_accuracy": 0.9857142857142858,
-      "eval_loss": 0.028362158685922623,
-      "eval_runtime": 5.2306,
-      "eval_samples_per_second": 13.383,
-      "eval_steps_per_second": 0.956,
-      "step": 60
+      "epoch": 12.842105263157894,
+      "eval_accuracy": 0.9571428571428572,
+      "eval_loss": 0.16158795356750488,
+      "eval_runtime": 5.284,
+      "eval_samples_per_second": 13.248,
+      "eval_steps_per_second": 0.946,
+      "step": 61
     },
     {
-      "epoch": 12.631578947368421,
-      "step": 60,
-      "total_flos": 2.993942051666657e+17,
-      "train_loss": 0.30815972487131754,
-      "train_runtime": 414.0498,
-      "train_samples_per_second": 10.868,
-      "train_steps_per_second": 0.145
+      "epoch": 13.894736842105264,
+      "eval_accuracy": 0.9571428571428572,
+      "eval_loss": 0.2285740226507187,
+      "eval_runtime": 5.3185,
+      "eval_samples_per_second": 13.161,
+      "eval_steps_per_second": 0.94,
+      "step": 66
+    },
+    {
+      "epoch": 14.736842105263158,
+      "grad_norm": 1.6222240924835205,
+      "learning_rate": 6.944444444444445e-06,
+      "loss": 0.0151,
+      "step": 70
+    },
+    {
+      "epoch": 14.947368421052632,
+      "eval_accuracy": 0.9571428571428572,
+      "eval_loss": 0.13689564168453217,
+      "eval_runtime": 5.3199,
+      "eval_samples_per_second": 13.158,
+      "eval_steps_per_second": 0.94,
+      "step": 71
+    },
+    {
+      "epoch": 16.0,
+      "eval_accuracy": 0.9571428571428572,
+      "eval_loss": 0.21541821956634521,
+      "eval_runtime": 5.3461,
+      "eval_samples_per_second": 13.094,
+      "eval_steps_per_second": 0.935,
+      "step": 76
+    },
+    {
+      "epoch": 16.842105263157894,
+      "grad_norm": 17.900920867919922,
+      "learning_rate": 0.0,
+      "loss": 0.0455,
+      "step": 80
+    },
+    {
+      "epoch": 16.842105263157894,
+      "eval_accuracy": 0.9428571428571428,
+      "eval_loss": 0.2586957514286041,
+      "eval_runtime": 5.2529,
+      "eval_samples_per_second": 13.326,
+      "eval_steps_per_second": 0.952,
+      "step": 80
+    },
+    {
+      "epoch": 16.842105263157894,
+      "step": 80,
+      "total_flos": 3.991922735555543e+17,
+      "train_loss": 0.10449823420494794,
+      "train_runtime": 538.8903,
+      "train_samples_per_second": 11.134,
+      "train_steps_per_second": 0.148
     }
   ],
   "logging_steps": 10,
-  "max_steps": 60,
+  "max_steps": 80,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 15,
+  "num_train_epochs": 20,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -194,7 +244,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.993942051666657e+17,
+  "total_flos": 3.991922735555543e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:124ad3a46ceaa8e0ec6490e95d380bdc480569c4d7708263d876031fcbfd9e2a
+oid sha256:573b1e7474596313ef9a434a86ed6dc105467a7fe304ddb816a9f274206a43b3
 size 5112