Model save

README.md

@@ -2,13 +2,9 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -19,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
 - Loss: 0.5028
 - Rewards/chosen: -0.9469
@@ -64,12 +60,12 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
-|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.5545        | 0.21  | 100  | -1.3212       | -1.0287         | -312.0799    | -374.3159      | 0.5658          | 0.7188             | -0.4953        | 0.6264          | -1.1217          |
-| 0.5026        | 0.42  | 200  | -0.1773       | 0.5190          | -352.4985    | -439.3264      | 0.5202          | 0.7461             | -0.8995        | 0.8723          | -1.7718          |
-| 0.5106        | 0.63  | 300  | 0.0862        | 0.9099          | -342.0043    | -424.9976      | 0.5104          | 0.7656             | -0.7946        | 0.8339          | -1.6285          |
-| 0.4859        | 0.84  | 400  | 0.7818        | 1.7438          | -360.3139    | -457.9452      | 0.5031          | 0.7578             | -0.9777        | 0.9803          | -1.9580          |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5545        | 0.21  | 100  | 0.5658          | -0.4953        | -1.1217          | 0.7188             | 0.6264          | -374.3159      | -312.0799    | -1.0287         | -1.3212       |
+| 0.5026        | 0.42  | 200  | 0.5202          | -0.8995        | -1.7718          | 0.7461             | 0.8723          | -439.3264      | -352.4985    | 0.5190          | -0.1773       |
+| 0.5106        | 0.63  | 300  | 0.5104          | -0.7946        | -1.6285          | 0.7656             | 0.8339          | -424.9976      | -342.0043    | 0.9099          | 0.0862        |
+| 0.4859        | 0.84  | 400  | 0.5031          | -0.9777        | -1.9580          | 0.7578             | 0.9803          | -457.9452      | -360.3139    | 1.7438          | 0.7818        |
 
 
 ### Framework versions

all_results.json

@@ -9,13 +9,13 @@
     "eval_rewards/chosen": -0.9468507170677185,
     "eval_rewards/margins": 0.946345865726471,
     "eval_rewards/rejected": -1.8931965827941895,
-    "eval_runtime": 87.2102,
+    "eval_runtime": 88.2311,
     "eval_samples": 2000,
-    "eval_samples_per_second": 22.933,
-    "eval_steps_per_second": 0.367,
-    "train_loss": 0.07998354963677698,
-    "train_runtime": 779.5821,
+    "eval_samples_per_second": 22.668,
+    "eval_steps_per_second": 0.363,
+    "train_loss": 0.5366686437918052,
+    "train_runtime": 5259.7251,
     "train_samples": 61135,
-    "train_samples_per_second": 78.42,
-    "train_steps_per_second": 0.613
+    "train_samples_per_second": 11.623,
+    "train_steps_per_second": 0.091
 }

eval_results.json

@@ -9,8 +9,8 @@
     "eval_rewards/chosen": -0.9468507170677185,
     "eval_rewards/margins": 0.946345865726471,
     "eval_rewards/rejected": -1.8931965827941895,
-    "eval_runtime": 87.2102,
+    "eval_runtime": 88.2311,
     "eval_samples": 2000,
-    "eval_samples_per_second": 22.933,
-    "eval_steps_per_second": 0.367
+    "eval_samples_per_second": 22.668,
+    "eval_steps_per_second": 0.363
 }

runs/Jan09_05-05-41_ip-26-0-161-142/events.out.tfevents.1704776777.ip-26-0-161-142.2982297.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1068be05be4b4c4c83692d6d4d75f6deee09d6a702a2c7606b08376b742f96aa
-size 33330
+oid sha256:45086008b51ce161ef9703ada557dbf4c3cb439d62224dfdcab11d57f3f06d19
+size 38122

runs/Jan09_05-05-41_ip-26-0-161-142/events.out.tfevents.1704782125.ip-26-0-161-142.2982297.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a80cc3e09600ca8513a0fa3bf27018544c48561c7d7ab1eb12474aff0b3f0a2
+size 828

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.07998354963677698,
-    "train_runtime": 779.5821,
+    "train_loss": 0.5366686437918052,
+    "train_runtime": 5259.7251,
     "train_samples": 61135,
-    "train_samples_per_second": 78.42,
-    "train_steps_per_second": 0.613
+    "train_samples_per_second": 11.623,
+    "train_steps_per_second": 0.091
 }

trainer_state.json

@@ -173,9 +173,9 @@
       "eval_rewards/chosen": -0.49532508850097656,
       "eval_rewards/margins": 0.6263692378997803,
       "eval_rewards/rejected": -1.1216944456100464,
-      "eval_runtime": 88.7193,
-      "eval_samples_per_second": 22.543,
-      "eval_steps_per_second": 0.361,
+      "eval_runtime": 87.9008,
+      "eval_samples_per_second": 22.753,
+      "eval_steps_per_second": 0.364,
       "step": 100
     },
     {
@@ -329,9 +329,9 @@
       "eval_rewards/chosen": -0.899510383605957,
       "eval_rewards/margins": 0.8722902536392212,
       "eval_rewards/rejected": -1.7718006372451782,
-      "eval_runtime": 88.5484,
-      "eval_samples_per_second": 22.587,
-      "eval_steps_per_second": 0.361,
+      "eval_runtime": 87.889,
+      "eval_samples_per_second": 22.756,
+      "eval_steps_per_second": 0.364,
       "step": 200
     },
     {
@@ -485,9 +485,9 @@
       "eval_rewards/chosen": -0.7945692539215088,
       "eval_rewards/margins": 0.8339425921440125,
       "eval_rewards/rejected": -1.628511905670166,
-      "eval_runtime": 88.6653,
-      "eval_samples_per_second": 22.557,
-      "eval_steps_per_second": 0.361,
+      "eval_runtime": 88.1333,
+      "eval_samples_per_second": 22.693,
+      "eval_steps_per_second": 0.363,
       "step": 300
     },
     {
@@ -641,8 +641,8 @@
       "eval_rewards/chosen": -0.9776647090911865,
       "eval_rewards/margins": 0.9803228974342346,
       "eval_rewards/rejected": -1.9579875469207764,
-      "eval_runtime": 88.4827,
-      "eval_samples_per_second": 22.603,
+      "eval_runtime": 88.4314,
+      "eval_samples_per_second": 22.616,
       "eval_steps_per_second": 0.362,
       "step": 400
     },
@@ -748,10 +748,10 @@
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.07998354963677698,
-      "train_runtime": 779.5821,
-      "train_samples_per_second": 78.42,
-      "train_steps_per_second": 0.613
+      "train_loss": 0.5366686437918052,
+      "train_runtime": 5259.7251,
+      "train_samples_per_second": 11.623,
+      "train_steps_per_second": 0.091
     }
   ],
   "logging_steps": 10,