End of training

README.md

@@ -2,9 +2,15 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
+- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
+- trl
+- dpo
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full-ultrabin-high-margin-3-epochs
   results: []
@@ -15,17 +21,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full-ultrabin-high-margin-3-epochs
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5884
-- Rewards/chosen: -2.9614
-- Rewards/rejected: -4.9020
+- Loss: 0.5894
+- Rewards/chosen: -2.9914
+- Rewards/rejected: -4.9379
 - Rewards/accuracies: 0.7578
-- Rewards/margins: 1.9406
-- Logps/rejected: -752.8591
-- Logps/chosen: -558.7693
-- Logits/rejected: 3.7820
-- Logits/chosen: 2.5008
+- Rewards/margins: 1.9464
+- Logps/rejected: -756.4492
+- Logps/chosen: -561.7738
+- Logits/rejected: 3.7803
+- Logits/chosen: 2.4993
 
 ## Model description
 

all_results.json

@@ -1,5 +1,18 @@
 {
     "epoch": 2.989547038327526,
+    "eval_logits/chosen": 2.499317169189453,
+    "eval_logits/rejected": 3.780329704284668,
+    "eval_logps/chosen": -561.7738037109375,
+    "eval_logps/rejected": -756.44921875,
+    "eval_loss": 0.5893987417221069,
+    "eval_rewards/accuracies": 0.7578125,
+    "eval_rewards/chosen": -2.991438865661621,
+    "eval_rewards/margins": 1.9464330673217773,
+    "eval_rewards/rejected": -4.93787145614624,
+    "eval_runtime": 104.5148,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 19.136,
+    "eval_steps_per_second": 0.306,
     "total_flos": 0.0,
     "train_loss": 0.22287911155840734,
     "train_runtime": 11425.5472,

config.json

@@ -22,6 +22,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.44.0.dev0",
-  "use_cache": false,
+  "use_cache": true,
   "vocab_size": 32000
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 2.989547038327526,
+    "eval_logits/chosen": 2.499317169189453,
+    "eval_logits/rejected": 3.780329704284668,
+    "eval_logps/chosen": -561.7738037109375,
+    "eval_logps/rejected": -756.44921875,
+    "eval_loss": 0.5893987417221069,
+    "eval_rewards/accuracies": 0.7578125,
+    "eval_rewards/chosen": -2.991438865661621,
+    "eval_rewards/margins": 1.9464330673217773,
+    "eval_rewards/rejected": -4.93787145614624,
+    "eval_runtime": 104.5148,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 19.136,
+    "eval_steps_per_second": 0.306
+}