Training in progress, step 18600

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97e8843eb054c3ad74976f86442c8477d9387361a93f5b5aa761f197a5bc9063
+oid sha256:5ee7bcf91faa799f3b50658aff33fa5ee549302575087b3dff52ae4b72610fef
 size 13982248

runs/Apr12_11-58-09_f044666cc877/events.out.tfevents.1712923163.f044666cc877.110.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9beca122599c177ea752ce3b4320cde847f76d9c8733a1ab24ed8ad19c4110e7
-size 201717
+oid sha256:771208a6e89f60fbedab34929f3248a1b91352640ffad51f4d5bbdbe3d80ef43
+size 201932

wandb/debug-internal.log

@@ -13909,3 +13909,16 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 17:26:27,641 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 17:26:27,642 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 17:26:27,885 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:26:29,886 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:26:32,409 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:37,410 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:42,411 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:42,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:26:42,511 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:26:42,515 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:26:47,599 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:49,432 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:26:49,433 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:26:49,433 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:26:49,434 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:26:49,893 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json

wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5458, "train/grad_norm": 19.47591209411621, "train/learning_rate": 6.580644556884702e-07, "train/epoch": 0.26, "train/global_step": 18580, "_timestamp": 1712942787.6394272, "_runtime": 19596.04845714569, "_step": 928}
+{"train/loss": 3.5333, "train/grad_norm": 31.87932014465332, "train/learning_rate": 6.397368838268497e-07, "train/epoch": 0.26, "train/global_step": 18600, "_timestamp": 1712942809.4317503, "_runtime": 19617.84078025818, "_step": 929}

wandb/run-20240412_115951-o0z5y86z/logs/debug-internal.log

@@ -13909,3 +13909,16 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 17:26:27,641 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 17:26:27,642 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 17:26:27,885 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:26:29,886 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:26:32,409 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:37,410 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:42,411 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:42,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:26:42,511 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:26:42,515 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:26:47,599 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:49,432 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:26:49,433 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:26:49,433 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:26:49,434 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:26:49,893 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json