train model

scripts/model.yaml

@@ -44,10 +44,11 @@ resume: "auto"
 # Data-related arguments. If not provided, the default is ``litgpt.data.TinyLlama``.
 data:
   class_path: LitData
+  
   init_args:
     data_path: "../data/"
-    # num_workers: 16
-    num_workers: 3
+    num_workers: 16
+    # num_workers: 3
 
 # Training-related arguments. See ``litgpt.args.TrainArgs`` for details
 train:
@@ -61,8 +62,8 @@ train:
   global_batch_size: 512
 
   # Number of samples per data-parallel rank (type: int, default: 4)
-  # micro_batch_size: 16
-  micro_batch_size: 14
+  micro_batch_size: 16
+  # micro_batch_size: 14
 
   # Number of iterations with learning rate warmup active (type: int, default: 2000)
   lr_warmup_steps: 2000
@@ -108,9 +109,9 @@ eval:
 
 # Optimizer-related arguments
 optimizer:
-  class_path: torch.optim.AdamW
+  # class_path: torch.optim.AdamW
   # class_path: bitsandbytes.optim.PagedAdamW
-  # class_path: bitsandbytes.optim.AdamW8bit
+  class_path: bitsandbytes.optim.AdamW8bit
   # class_path: bitsandbytes.optim.PagedAdamW8bit
   
   init_args:

scripts/requirements-lit.in

@@ -5,6 +5,7 @@ jinja2
 transformers
 bitsandbytes
 wandb
-litgpt[all]
+# litgpt[all]
+litgpt[all] @ git+https://github.com/mtasic85/litgpt.git
 litdata
 grokadamw