Model save

README.md

@@ -2,13 +2,12 @@
 license: mit
 library_name: peft
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - sft
+- generated_from_trainer
 base_model: HuggingFaceH4/zephyr-7b-beta
 datasets:
-- erbacher/rag-and-roll
+- generator
 model-index:
 - name: zephyr-rag-agent
   results: []
@@ -19,9 +18,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-rag-agent
 
-This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) on the erbacher/rag-and-roll dataset.
+This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.1623
+- Loss: 1.1829
 
 ## Model description
 
@@ -40,19 +39,19 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.0002
-- train_batch_size: 2
-- eval_batch_size: 2
+- learning_rate: 4e-05
+- train_batch_size: 4
+- eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 2
 - gradient_accumulation_steps: 16
-- total_train_batch_size: 64
-- total_eval_batch_size: 4
+- total_train_batch_size: 128
+- total_eval_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 3
+- num_epochs: 10
 
 ### Training results
 
@@ -61,6 +60,10 @@ The following hyperparameters were used during training:
 | 1.1559        | 0.99  | 27   | 1.1650          |
 | 1.0887        | 1.98  | 54   | 1.1555          |
 | 1.0566        | 2.97  | 81   | 1.1623          |
+| 1.0264        | 6.95  | 91   | 1.1689          |
+| 0.9977        | 7.97  | 105  | 1.1779          |
+| 0.9808        | 9.0   | 119  | 1.1820          |
+| 0.9791        | 9.8   | 130  | 1.1829          |
 
 
 ### Framework versions

all_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 2.97,
-    "eval_loss": 1.1623399257659912,
-    "eval_runtime": 279.5897,
+    "epoch": 9.8,
+    "eval_loss": 1.1828593015670776,
+    "eval_runtime": 207.1428,
     "eval_samples": 716,
-    "eval_samples_per_second": 2.167,
-    "eval_steps_per_second": 0.544,
-    "train_loss": 1.1298090528558802,
-    "train_runtime": 14861.7425,
+    "eval_samples_per_second": 2.926,
+    "eval_steps_per_second": 0.367,
+    "train_loss": 0.37551442659818207,
+    "train_runtime": 8428.1057,
     "train_samples": 1922,
-    "train_samples_per_second": 0.352,
-    "train_steps_per_second": 0.005
+    "train_samples_per_second": 2.07,
+    "train_steps_per_second": 0.015
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.97,
-    "eval_loss": 1.1623399257659912,
-    "eval_runtime": 279.5897,
+    "epoch": 9.8,
+    "eval_loss": 1.1828593015670776,
+    "eval_runtime": 207.1428,
     "eval_samples": 716,
-    "eval_samples_per_second": 2.167,
-    "eval_steps_per_second": 0.544
+    "eval_samples_per_second": 2.926,
+    "eval_steps_per_second": 0.367
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.97,
-    "train_loss": 1.1298090528558802,
-    "train_runtime": 14861.7425,
+    "epoch": 9.8,
+    "train_loss": 0.37551442659818207,
+    "train_runtime": 8428.1057,
     "train_samples": 1922,
-    "train_samples_per_second": 0.352,
-    "train_steps_per_second": 0.005
+    "train_samples_per_second": 2.07,
+    "train_steps_per_second": 0.015
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.965675057208238,
+  "epoch": 9.799086757990867,
   "eval_steps": 500,
-  "global_step": 81,
+  "global_step": 130,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -135,21 +135,113 @@
       "step": 81
     },
     {
-      "epoch": 2.97,
-      "step": 81,
-      "total_flos": 173916983132160.0,
-      "train_loss": 1.1298090528558802,
-      "train_runtime": 14861.7425,
-      "train_samples_per_second": 0.352,
-      "train_steps_per_second": 0.005
+      "epoch": 6.51,
+      "learning_rate": 6.453951129574644e-05,
+      "loss": 1.0309,
+      "step": 85
+    },
+    {
+      "epoch": 6.88,
+      "learning_rate": 5.234312799786921e-05,
+      "loss": 1.0264,
+      "step": 90
+    },
+    {
+      "epoch": 6.95,
+      "eval_loss": 1.168891191482544,
+      "eval_runtime": 208.2969,
+      "eval_samples_per_second": 2.909,
+      "eval_steps_per_second": 0.365,
+      "step": 91
+    },
+    {
+      "epoch": 7.24,
+      "learning_rate": 4.100445599768774e-05,
+      "loss": 1.0037,
+      "step": 95
+    },
+    {
+      "epoch": 7.61,
+      "learning_rate": 3.072756464904006e-05,
+      "loss": 1.0055,
+      "step": 100
+    },
+    {
+      "epoch": 7.97,
+      "learning_rate": 2.1697413758237784e-05,
+      "loss": 0.9977,
+      "step": 105
+    },
+    {
+      "epoch": 7.97,
+      "eval_loss": 1.1778818368911743,
+      "eval_runtime": 208.3439,
+      "eval_samples_per_second": 2.909,
+      "eval_steps_per_second": 0.365,
+      "step": 105
+    },
+    {
+      "epoch": 8.34,
+      "learning_rate": 1.4076524743778319e-05,
+      "loss": 0.9801,
+      "step": 110
+    },
+    {
+      "epoch": 8.7,
+      "learning_rate": 8.002055634117578e-06,
+      "loss": 0.9808,
+      "step": 115
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 1.1819604635238647,
+      "eval_runtime": 208.427,
+      "eval_samples_per_second": 2.907,
+      "eval_steps_per_second": 0.365,
+      "step": 119
+    },
+    {
+      "epoch": 9.07,
+      "learning_rate": 3.5833325466437694e-06,
+      "loss": 0.9842,
+      "step": 120
+    },
+    {
+      "epoch": 9.43,
+      "learning_rate": 8.998820754091531e-07,
+      "loss": 0.9813,
+      "step": 125
+    },
+    {
+      "epoch": 9.8,
+      "learning_rate": 0.0,
+      "loss": 0.9791,
+      "step": 130
+    },
+    {
+      "epoch": 9.8,
+      "eval_loss": 1.1828593015670776,
+      "eval_runtime": 208.4043,
+      "eval_samples_per_second": 2.908,
+      "eval_steps_per_second": 0.365,
+      "step": 130
+    },
+    {
+      "epoch": 9.8,
+      "step": 130,
+      "total_flos": 384243276447744.0,
+      "train_loss": 0.37551442659818207,
+      "train_runtime": 8428.1057,
+      "train_samples_per_second": 2.07,
+      "train_steps_per_second": 0.015
     }
   ],
   "logging_steps": 5,
-  "max_steps": 81,
+  "max_steps": 130,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 10,
   "save_steps": 500,
-  "total_flos": 173916983132160.0,
+  "total_flos": 384243276447744.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null