Model save

README.md

@@ -2,15 +2,10 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full-ultrabin-high-bleu
   results: []
@@ -21,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full-ultrabin-high-bleu
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6282
-- Rewards/chosen: -0.2999
-- Rewards/rejected: -0.5424
-- Rewards/accuracies: 0.7031
-- Rewards/margins: 0.2425
-- Logps/rejected: -316.9008
-- Logps/chosen: -292.6180
-- Logits/rejected: -2.2318
-- Logits/chosen: -2.2813
+- Loss: 0.6311
+- Rewards/chosen: -0.3272
+- Rewards/rejected: -0.5547
+- Rewards/accuracies: 0.6875
+- Rewards/margins: 0.2276
+- Logps/rejected: -318.1367
+- Logps/chosen: -295.3459
+- Logits/rejected: -2.3484
+- Logits/chosen: -2.3932
 
 ## Model description
 
@@ -68,8 +63,8 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.66          | 0.3484 | 50   | 0.6513          | -0.1132        | -0.2374          | 0.6641             | 0.1242          | -286.3991      | -273.9513    | -2.4980         | -2.5369       |
-| 0.6376        | 0.6969 | 100  | 0.6329          | -0.2639        | -0.4887          | 0.6875             | 0.2248          | -311.5294      | -289.0153    | -2.2959         | -2.3456       |
+| 0.6597        | 0.3484 | 50   | 0.6513          | -0.1047        | -0.2299          | 0.6641             | 0.1252          | -285.6537      | -273.1035    | -2.4969         | -2.5355       |
+| 0.6383        | 0.6969 | 100  | 0.6311          | -0.3272        | -0.5547          | 0.6875             | 0.2276          | -318.1367      | -295.3459    | -2.3484         | -2.3932       |
 
 
 ### Framework versions

all_results.json

@@ -14,9 +14,9 @@
     "eval_samples_per_second": 19.065,
     "eval_steps_per_second": 0.305,
     "total_flos": 0.0,
-    "train_loss": 0.6509068312344851,
-    "train_runtime": 3880.8302,
+    "train_loss": 0.6505647914392965,
+    "train_runtime": 3852.398,
     "train_samples": 18340,
-    "train_samples_per_second": 4.726,
+    "train_samples_per_second": 4.761,
     "train_steps_per_second": 0.037
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b2b9259176ea21cd446935814aeeb2cbd2c26189b316fc1111f2c292a1ada18
+oid sha256:9378636ec91b3e02e5687373b6eec44210ca5fc9ce51802efdecb5468d62118e
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54164d9bfaf5d4fa59c2cdff47ac3d0b8ceba43b2b18c0ec87bc3225001a9c42
+oid sha256:18e62239c73ac4e9d825ec302bfc5d88815699688cc69adf69aeec4c380376a4
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5581f68d1158abd35ff5b749a7461714f51a7895e6f7ab6de27643ccc9c44e02
+oid sha256:81d7e8997d64033b8232ad6f56b4654c23d7c466f9b3ca9b049e64261e4af8e6
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9965156794425087,
     "total_flos": 0.0,
-    "train_loss": 0.6509068312344851,
-    "train_runtime": 3880.8302,
+    "train_loss": 0.6505647914392965,
+    "train_runtime": 3852.398,
     "train_samples": 18340,
-    "train_samples_per_second": 4.726,
+    "train_samples_per_second": 4.761,
     "train_steps_per_second": 0.037
 }

trainer_state.json

@@ -10,253 +10,253 @@
   "log_history": [
     {
       "epoch": 0.06968641114982578,
-      "grad_norm": 7.974906667324644,
+      "grad_norm": 7.942402444587964,
       "learning_rate": 3.333333333333333e-07,
-      "logits/chosen": -2.747501850128174,
-      "logits/rejected": -2.7370193004608154,
-      "logps/chosen": -291.8797302246094,
-      "logps/rejected": -281.50189208984375,
+      "logits/chosen": -2.747418165206909,
+      "logits/rejected": -2.7369940280914307,
+      "logps/chosen": -291.8316650390625,
+      "logps/rejected": -281.5037536621094,
       "loss": 0.693,
-      "rewards/accuracies": 0.3812499940395355,
-      "rewards/chosen": -0.00011452056060079485,
-      "rewards/margins": -3.700270826811902e-05,
-      "rewards/rejected": -7.751785597065464e-05,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": 0.0003662299714051187,
+      "rewards/margins": 0.0004625393485184759,
+      "rewards/rejected": -9.630931162973866e-05,
       "step": 10
     },
     {
       "epoch": 0.13937282229965156,
-      "grad_norm": 8.1980266894883,
+      "grad_norm": 8.199051257817597,
       "learning_rate": 4.981198836496775e-07,
-      "logits/chosen": -2.7630066871643066,
-      "logits/rejected": -2.748826026916504,
-      "logps/chosen": -298.94158935546875,
-      "logps/rejected": -320.3427429199219,
-      "loss": 0.6902,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.008970660157501698,
-      "rewards/margins": 0.006611389108002186,
-      "rewards/rejected": 0.002359271515160799,
+      "logits/chosen": -2.7628424167633057,
+      "logits/rejected": -2.7486491203308105,
+      "logps/chosen": -298.9639587402344,
+      "logps/rejected": -320.4031066894531,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.008746958337724209,
+      "rewards/margins": 0.006991321686655283,
+      "rewards/rejected": 0.0017556389793753624,
       "step": 20
     },
     {
       "epoch": 0.20905923344947736,
-      "grad_norm": 8.031029605438988,
+      "grad_norm": 8.009396280118287,
       "learning_rate": 4.832481997086846e-07,
-      "logits/chosen": -2.7137038707733154,
-      "logits/rejected": -2.7142727375030518,
-      "logps/chosen": -296.2449951171875,
-      "logps/rejected": -289.28814697265625,
+      "logits/chosen": -2.713383436203003,
+      "logits/rejected": -2.714205503463745,
+      "logps/chosen": -296.2840881347656,
+      "logps/rejected": -289.38421630859375,
       "loss": 0.681,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.040586382150650024,
-      "rewards/margins": 0.03953887149691582,
-      "rewards/rejected": 0.0010475128656253219,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.04019603878259659,
+      "rewards/margins": 0.04010898619890213,
+      "rewards/rejected": 8.704829815542325e-05,
       "step": 30
     },
     {
       "epoch": 0.2787456445993031,
-      "grad_norm": 9.014305613217058,
+      "grad_norm": 8.916455655557288,
       "learning_rate": 4.543962032878959e-07,
-      "logits/chosen": -2.679992198944092,
-      "logits/rejected": -2.675081253051758,
-      "logps/chosen": -298.83929443359375,
-      "logps/rejected": -318.2642822265625,
+      "logits/chosen": -2.679121732711792,
+      "logits/rejected": -2.674286365509033,
+      "logps/chosen": -298.8094177246094,
+      "logps/rejected": -318.2736511230469,
       "loss": 0.6637,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.003969565033912659,
-      "rewards/margins": 0.06901098042726517,
-      "rewards/rejected": -0.07298054546117783,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.003670867532491684,
+      "rewards/margins": 0.06940315663814545,
+      "rewards/rejected": -0.07307400554418564,
       "step": 40
     },
     {
       "epoch": 0.34843205574912894,
-      "grad_norm": 10.923357206738944,
+      "grad_norm": 11.007582392023577,
       "learning_rate": 4.1329321073844413e-07,
-      "logits/chosen": -2.547337770462036,
-      "logits/rejected": -2.5558876991271973,
-      "logps/chosen": -272.10382080078125,
-      "logps/rejected": -269.55987548828125,
-      "loss": 0.66,
+      "logits/chosen": -2.5453474521636963,
+      "logits/rejected": -2.553844451904297,
+      "logps/chosen": -272.39227294921875,
+      "logps/rejected": -269.97125244140625,
+      "loss": 0.6597,
       "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.13082024455070496,
-      "rewards/margins": 0.06251106411218643,
-      "rewards/rejected": -0.1933313012123108,
+      "rewards/chosen": -0.1337047815322876,
+      "rewards/margins": 0.06374003738164902,
+      "rewards/rejected": -0.1974448263645172,
       "step": 50
     },
     {
       "epoch": 0.34843205574912894,
-      "eval_logits/chosen": -2.536876678466797,
-      "eval_logits/rejected": -2.4980242252349854,
-      "eval_logps/chosen": -273.9512634277344,
-      "eval_logps/rejected": -286.3990783691406,
-      "eval_loss": 0.6512594819068909,
+      "eval_logits/chosen": -2.5355470180511475,
+      "eval_logits/rejected": -2.496910572052002,
+      "eval_logps/chosen": -273.103515625,
+      "eval_logps/rejected": -285.6536865234375,
+      "eval_loss": 0.6512949466705322,
       "eval_rewards/accuracies": 0.6640625,
-      "eval_rewards/chosen": -0.11321306228637695,
-      "eval_rewards/margins": 0.12415696680545807,
-      "eval_rewards/rejected": -0.23737002909183502,
-      "eval_runtime": 103.5252,
-      "eval_samples_per_second": 19.319,
-      "eval_steps_per_second": 0.309,
+      "eval_rewards/chosen": -0.1047358363866806,
+      "eval_rewards/margins": 0.1251804083585739,
+      "eval_rewards/rejected": -0.22991624474525452,
+      "eval_runtime": 105.224,
+      "eval_samples_per_second": 19.007,
+      "eval_steps_per_second": 0.304,
       "step": 50
     },
     {
       "epoch": 0.4181184668989547,
-      "grad_norm": 11.450114313746024,
+      "grad_norm": 11.130762557977343,
       "learning_rate": 3.624028324136517e-07,
-      "logits/chosen": -2.5856661796569824,
-      "logits/rejected": -2.5894103050231934,
-      "logps/chosen": -315.35540771484375,
-      "logps/rejected": -320.2020263671875,
-      "loss": 0.6507,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.12617941200733185,
-      "rewards/margins": 0.15071702003479004,
-      "rewards/rejected": -0.2768964171409607,
+      "logits/chosen": -2.5859858989715576,
+      "logits/rejected": -2.590031147003174,
+      "logps/chosen": -312.9039611816406,
+      "logps/rejected": -316.5061950683594,
+      "loss": 0.6515,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.10166505724191666,
+      "rewards/margins": 0.13827314972877502,
+      "rewards/rejected": -0.23993822932243347,
       "step": 60
     },
     {
       "epoch": 0.4878048780487805,
-      "grad_norm": 10.762410651532795,
+      "grad_norm": 10.367706676784401,
       "learning_rate": 3.047753100392174e-07,
-      "logits/chosen": -2.531461238861084,
-      "logits/rejected": -2.5228896141052246,
-      "logps/chosen": -301.96588134765625,
-      "logps/rejected": -300.63177490234375,
-      "loss": 0.6458,
+      "logits/chosen": -2.515353202819824,
+      "logits/rejected": -2.5070269107818604,
+      "logps/chosen": -303.077392578125,
+      "logps/rejected": -301.082763671875,
+      "loss": 0.6453,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.23531746864318848,
-      "rewards/margins": 0.1557210385799408,
-      "rewards/rejected": -0.3910384774208069,
+      "rewards/chosen": -0.2464326173067093,
+      "rewards/margins": 0.14911559224128723,
+      "rewards/rejected": -0.3955482840538025,
       "step": 70
     },
     {
       "epoch": 0.5574912891986062,
-      "grad_norm": 10.243677934319797,
+      "grad_norm": 10.328785377431698,
       "learning_rate": 2.4386469286927194e-07,
-      "logits/chosen": -2.531421184539795,
-      "logits/rejected": -2.5143864154815674,
-      "logps/chosen": -289.48651123046875,
-      "logps/rejected": -301.8067932128906,
-      "loss": 0.6511,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.187635600566864,
-      "rewards/margins": 0.1388576477766037,
-      "rewards/rejected": -0.3264932334423065,
+      "logits/chosen": -2.534677028656006,
+      "logits/rejected": -2.515333652496338,
+      "logps/chosen": -305.86944580078125,
+      "logps/rejected": -315.22369384765625,
+      "loss": 0.6525,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.3514646887779236,
+      "rewards/margins": 0.10919757932424545,
+      "rewards/rejected": -0.460662305355072,
       "step": 80
     },
     {
       "epoch": 0.627177700348432,
-      "grad_norm": 10.605767289766804,
+      "grad_norm": 10.256439047308396,
       "learning_rate": 1.8332181063127542e-07,
-      "logits/chosen": -2.370906114578247,
-      "logits/rejected": -2.396063804626465,
-      "logps/chosen": -266.50262451171875,
-      "logps/rejected": -298.68292236328125,
-      "loss": 0.634,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.13985350728034973,
-      "rewards/margins": 0.18151375651359558,
-      "rewards/rejected": -0.3213672339916229,
+      "logits/chosen": -2.4058992862701416,
+      "logits/rejected": -2.4308979511260986,
+      "logps/chosen": -272.29986572265625,
+      "logps/rejected": -304.98779296875,
+      "loss": 0.6315,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.19782552123069763,
+      "rewards/margins": 0.18659016489982605,
+      "rewards/rejected": -0.38441571593284607,
       "step": 90
     },
     {
       "epoch": 0.6968641114982579,
-      "grad_norm": 11.921015517224783,
+      "grad_norm": 11.456138890836337,
       "learning_rate": 1.26775451942554e-07,
-      "logits/chosen": -2.410521984100342,
-      "logits/rejected": -2.4090819358825684,
-      "logps/chosen": -325.18603515625,
-      "logps/rejected": -335.88739013671875,
-      "loss": 0.6376,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.2180793583393097,
-      "rewards/margins": 0.21515274047851562,
-      "rewards/rejected": -0.4332321286201477,
+      "logits/chosen": -2.4451889991760254,
+      "logits/rejected": -2.447258472442627,
+      "logps/chosen": -332.0220031738281,
+      "logps/rejected": -343.72979736328125,
+      "loss": 0.6383,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2864390015602112,
+      "rewards/margins": 0.22521762549877167,
+      "rewards/rejected": -0.511656641960144,
       "step": 100
     },
     {
       "epoch": 0.6968641114982579,
-      "eval_logits/chosen": -2.345639705657959,
-      "eval_logits/rejected": -2.2959439754486084,
-      "eval_logps/chosen": -289.0152893066406,
-      "eval_logps/rejected": -311.52935791015625,
-      "eval_loss": 0.6328787803649902,
+      "eval_logits/chosen": -2.393218517303467,
+      "eval_logits/rejected": -2.3483967781066895,
+      "eval_logps/chosen": -295.345947265625,
+      "eval_logps/rejected": -318.13665771484375,
+      "eval_loss": 0.6311340928077698,
       "eval_rewards/accuracies": 0.6875,
-      "eval_rewards/chosen": -0.26385369896888733,
-      "eval_rewards/margins": 0.22481901943683624,
-      "eval_rewards/rejected": -0.4886727035045624,
-      "eval_runtime": 105.3794,
-      "eval_samples_per_second": 18.979,
-      "eval_steps_per_second": 0.304,
+      "eval_rewards/chosen": -0.3271602392196655,
+      "eval_rewards/margins": 0.22758512198925018,
+      "eval_rewards/rejected": -0.5547453165054321,
+      "eval_runtime": 103.8391,
+      "eval_samples_per_second": 19.261,
+      "eval_steps_per_second": 0.308,
       "step": 100
     },
     {
       "epoch": 0.7665505226480837,
-      "grad_norm": 10.917825147199611,
+      "grad_norm": 15.049705548114918,
       "learning_rate": 7.761486381573326e-08,
-      "logits/chosen": -2.425245761871338,
-      "logits/rejected": -2.4225878715515137,
-      "logps/chosen": -340.6777038574219,
-      "logps/rejected": -376.40008544921875,
-      "loss": 0.6263,
+      "logits/chosen": -2.472832679748535,
+      "logits/rejected": -2.4680256843566895,
+      "logps/chosen": -347.2950439453125,
+      "logps/rejected": -383.56903076171875,
+      "loss": 0.6223,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.29727238416671753,
-      "rewards/margins": 0.22700247168540955,
-      "rewards/rejected": -0.5242748856544495,
+      "rewards/chosen": -0.36344587802886963,
+      "rewards/margins": 0.232518270611763,
+      "rewards/rejected": -0.5959641933441162,
       "step": 110
     },
     {
       "epoch": 0.8362369337979094,
-      "grad_norm": 11.561681770035124,
+      "grad_norm": 11.65420047853757,
       "learning_rate": 3.878660868757322e-08,
-      "logits/chosen": -2.329202175140381,
-      "logits/rejected": -2.3225998878479004,
-      "logps/chosen": -323.3353271484375,
-      "logps/rejected": -341.76959228515625,
-      "loss": 0.6324,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.2884915769100189,
-      "rewards/margins": 0.22677147388458252,
-      "rewards/rejected": -0.5152631402015686,
+      "logits/chosen": -2.3938543796539307,
+      "logits/rejected": -2.3905398845672607,
+      "logps/chosen": -329.33978271484375,
+      "logps/rejected": -346.9275817871094,
+      "loss": 0.6335,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.34853675961494446,
+      "rewards/margins": 0.21830621361732483,
+      "rewards/rejected": -0.5668429136276245,
       "step": 120
     },
     {
       "epoch": 0.9059233449477352,
-      "grad_norm": 11.514501964616796,
+      "grad_norm": 11.07371443957527,
       "learning_rate": 1.261795485174083e-08,
-      "logits/chosen": -2.403071165084839,
-      "logits/rejected": -2.402184009552002,
-      "logps/chosen": -327.9507141113281,
-      "logps/rejected": -337.8323059082031,
-      "loss": 0.6248,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.27428099513053894,
-      "rewards/margins": 0.22150692343711853,
-      "rewards/rejected": -0.49578791856765747,
+      "logits/chosen": -2.4500081539154053,
+      "logits/rejected": -2.451570510864258,
+      "logps/chosen": -330.7276306152344,
+      "logps/rejected": -342.2732849121094,
+      "loss": 0.626,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3020505905151367,
+      "rewards/margins": 0.23814749717712402,
+      "rewards/rejected": -0.540198028087616,
       "step": 130
     },
     {
       "epoch": 0.975609756097561,
-      "grad_norm": 11.032500717539952,
+      "grad_norm": 11.70940252198527,
       "learning_rate": 6.773858303274482e-10,
-      "logits/chosen": -2.3575150966644287,
-      "logits/rejected": -2.346174716949463,
-      "logps/chosen": -298.3825988769531,
-      "logps/rejected": -313.13848876953125,
-      "loss": 0.6258,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.3439212739467621,
-      "rewards/margins": 0.12675735354423523,
-      "rewards/rejected": -0.4706786274909973,
+      "logits/chosen": -2.421135425567627,
+      "logits/rejected": -2.4089722633361816,
+      "logps/chosen": -300.1723937988281,
+      "logps/rejected": -315.6776123046875,
+      "loss": 0.6255,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.36181899905204773,
+      "rewards/margins": 0.13425084948539734,
+      "rewards/rejected": -0.49606984853744507,
       "step": 140
     },
     {
       "epoch": 0.9965156794425087,
       "step": 143,
       "total_flos": 0.0,
-      "train_loss": 0.6509068312344851,
-      "train_runtime": 3880.8302,
-      "train_samples_per_second": 4.726,
+      "train_loss": 0.6505647914392965,
+      "train_runtime": 3852.398,
+      "train_samples_per_second": 4.761,
       "train_steps_per_second": 0.037
     }
   ],