Training in progress, step 9380

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce3133fc8023dc9fec5e655ff4095b17d357c0b7879a0cb32cee9112f5995b98
+oid sha256:0bec6bca3c48b3e9f7ec1a052500985b06b331ed7b76a2c7795a8ebc958fc6fb
 size 13982248

runs/Apr12_07-40-41_e5a48bec8248/events.out.tfevents.1712907642.e5a48bec8248.443.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e7ead3a0fb12e65bd74f527130ae318c1894aa16805dfd09110c387f5d9e467
-size 104006
+oid sha256:9bf1e7e185bf066e7205ed3e7454f0413f1538a95e91218bc48f83777d5859a5
+size 104217

wandb/debug-internal.log

@@ -7722,3 +7722,19 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 10:51:45,419 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
 2024-04-12 10:51:45,690 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 10:51:46,691 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:51:47,691 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:51:47,994 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:51:47,994 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:51:49,057 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:51:54,058 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:51:59,059 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:52:02,691 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:52:02,994 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:52:02,994 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:52:04,121 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:52:07,599 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:52:07,601 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:52:07,601 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:52:07,601 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:52:07,698 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:52:09,417 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240412_074044-widovcn3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.4322, "train/grad_norm": 33.9267578125, "train/learning_rate": 1.7274514748802418e-05, "train/epoch": 0.75, "train/global_step": 9360, "_timestamp": 1712919105.0719736, "_runtime": 11460.564255475998, "_step": 467}
+{"train/loss": 3.443, "train/grad_norm": 29.3850040435791, "train/learning_rate": 1.7226483336120707e-05, "train/epoch": 0.75, "train/global_step": 9380, "_timestamp": 1712919127.5994036, "_runtime": 11483.091685533524, "_step": 468}

wandb/run-20240412_074044-widovcn3/logs/debug-internal.log

@@ -7722,3 +7722,19 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 10:51:45,419 DEBUG   SenderThread:485 [sender.py:send():379] send: stats
 2024-04-12 10:51:45,690 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
 2024-04-12 10:51:46,691 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/output.log
+2024-04-12 10:51:47,691 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:51:47,994 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:51:47,994 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:51:49,057 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:51:54,058 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:51:59,059 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:52:02,691 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 10:52:02,994 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 10:52:02,994 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 10:52:04,121 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 10:52:07,599 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 10:52:07,601 DEBUG   SenderThread:485 [sender.py:send():379] send: history
+2024-04-12 10:52:07,601 DEBUG   SenderThread:485 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 10:52:07,601 INFO    SenderThread:485 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 10:52:07,698 INFO    Thread-12 :485 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_074044-widovcn3/files/wandb-summary.json
+2024-04-12 10:52:09,417 DEBUG   HandlerThread:485 [handler.py:handle_request():146] handle_request: status_report