Model save

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6618
-- Rewards/chosen: 0.0116
-- Rewards/rejected: -0.0133
-- Rewards/accuracies: 0.6150
-- Rewards/margins: 0.0249
-- Logps/rejected: -262.4203
-- Logps/chosen: -281.8592
-- Logits/rejected: -2.8344
-- Logits/chosen: -2.8413
-- Use Label: 18709.0195
-- Pred Label: 13352.9805
+- Loss: 0.6741
+- Rewards/chosen: 0.0220
+- Rewards/rejected: -0.0242
+- Rewards/accuracies: 0.6680
+- Rewards/margins: 0.0462
+- Logps/rejected: -259.5420
+- Logps/chosen: -284.0372
+- Logits/rejected: -2.8454
+- Logits/chosen: -2.8419
+- Use Label: 0.0
+- Pred Label: 0.0
 
 ## Model description
 
@@ -49,10 +49,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 2
-- gradient_accumulation_steps: 8
+- num_devices: 4
+- gradient_accumulation_steps: 4
 - total_train_batch_size: 64
-- total_eval_batch_size: 8
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1
@@ -60,9 +60,9 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label  | Pred Label |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:----------:|:----------:|
-| 0.6628        | 1.0   | 955  | 0.6618          | 0.0116         | -0.0133          | 0.6150             | 0.0249          | -262.4203      | -281.8592    | -2.8344         | -2.8413       | 18206.0195 | 12855.9805 |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
+| 0.6744        | 1.0   | 955  | 0.6741          | 0.0220         | -0.0242          | 0.6680             | 0.0462          | -259.5420      | -284.0372    | -2.8454         | -2.8419       | 0.0       | 0.0        |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50dec1b84957e69e3addc889d603d42852492e7b6871ac38b15599e17d0a0147
+oid sha256:3de4b6a0728c8fbd836578308d9379c925334f979968e2be33179009eed76454
 size 218138576

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.841262102127075,
-    "eval_logits/rejected": -2.8343887329101562,
-    "eval_logps/chosen": -281.85919189453125,
-    "eval_logps/rejected": -262.4202880859375,
-    "eval_loss": 0.6618225574493408,
-    "eval_pred_label": 13352.98046875,
-    "eval_rewards/accuracies": 0.6150000095367432,
-    "eval_rewards/chosen": 0.011613711714744568,
-    "eval_rewards/margins": 0.02489962987601757,
-    "eval_rewards/rejected": -0.013285920023918152,
-    "eval_runtime": 834.3056,
+    "eval_logits/chosen": -2.8419151306152344,
+    "eval_logits/rejected": -2.845423936843872,
+    "eval_logps/chosen": -284.0372314453125,
+    "eval_logps/rejected": -259.5419921875,
+    "eval_loss": 0.6741092801094055,
+    "eval_pred_label": 0.0,
+    "eval_rewards/accuracies": 0.6679999828338623,
+    "eval_rewards/chosen": 0.02201448194682598,
+    "eval_rewards/margins": 0.04624143987894058,
+    "eval_rewards/rejected": -0.02422695979475975,
+    "eval_runtime": 465.3584,
     "eval_samples": 2000,
-    "eval_samples_per_second": 2.397,
-    "eval_steps_per_second": 0.3,
-    "eval_use_label": 18709.01953125,
-    "train_loss": 0.6705795382954063,
-    "train_runtime": 45840.6595,
+    "eval_samples_per_second": 4.298,
+    "eval_steps_per_second": 0.269,
+    "eval_use_label": 0.0,
+    "train_loss": 0.6817296707193264,
+    "train_runtime": 25631.2708,
     "train_samples": 61135,
-    "train_samples_per_second": 1.334,
-    "train_steps_per_second": 0.021
+    "train_samples_per_second": 2.385,
+    "train_steps_per_second": 0.037
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.841262102127075,
-    "eval_logits/rejected": -2.8343887329101562,
-    "eval_logps/chosen": -281.85919189453125,
-    "eval_logps/rejected": -262.4202880859375,
-    "eval_loss": 0.6618225574493408,
-    "eval_pred_label": 13352.98046875,
-    "eval_rewards/accuracies": 0.6150000095367432,
-    "eval_rewards/chosen": 0.011613711714744568,
-    "eval_rewards/margins": 0.02489962987601757,
-    "eval_rewards/rejected": -0.013285920023918152,
-    "eval_runtime": 834.3056,
+    "eval_logits/chosen": -2.8419151306152344,
+    "eval_logits/rejected": -2.845423936843872,
+    "eval_logps/chosen": -284.0372314453125,
+    "eval_logps/rejected": -259.5419921875,
+    "eval_loss": 0.6741092801094055,
+    "eval_pred_label": 0.0,
+    "eval_rewards/accuracies": 0.6679999828338623,
+    "eval_rewards/chosen": 0.02201448194682598,
+    "eval_rewards/margins": 0.04624143987894058,
+    "eval_rewards/rejected": -0.02422695979475975,
+    "eval_runtime": 465.3584,
     "eval_samples": 2000,
-    "eval_samples_per_second": 2.397,
-    "eval_steps_per_second": 0.3,
-    "eval_use_label": 18709.01953125
+    "eval_samples_per_second": 4.298,
+    "eval_steps_per_second": 0.269,
+    "eval_use_label": 0.0
 }

runs/Jan20_17-20-45_uclaml04.cs.ucla.edu/events.out.tfevents.1705800111.uclaml04.cs.ucla.edu.297358.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c57e845c5e45551fb4ce09d53dbfcebee11ff839380431898c267476990f0c41
-size 11823
+oid sha256:11b2ed24d7c5f72ee264779528d97138dd4794c77fded61dcf78550b01334188
+size 13024

runs/Jan20_17-20-45_uclaml04.cs.ucla.edu/events.out.tfevents.1705826207.uclaml04.cs.ucla.edu.297358.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2b21d40fd1081b2243755d435cc236c37d2506698b65209d5474a3efde6ef7
+size 935

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6705795382954063,
-    "train_runtime": 45840.6595,
+    "train_loss": 0.6817296707193264,
+    "train_runtime": 25631.2708,
     "train_samples": 61135,
-    "train_samples_per_second": 1.334,
-    "train_steps_per_second": 0.021
+    "train_samples_per_second": 2.385,
+    "train_steps_per_second": 0.037
 }

trainer_state.json

@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 5.208333333333333e-09,
-      "logits/chosen": -2.899709463119507,
-      "logits/rejected": -2.879509687423706,
-      "logps/chosen": -314.8815612792969,
-      "logps/rejected": -239.785888671875,
+      "logits/chosen": -2.980285167694092,
+      "logits/rejected": -2.87275767326355,
+      "logps/chosen": -313.4390563964844,
+      "logps/rejected": -236.1754150390625,
       "loss": 0.6931,
       "pred_label": 0.0,
       "rewards/accuracies": 0.0,
@@ -22,184 +22,184 @@
       "rewards/margins": 0.0,
       "rewards/rejected": 0.0,
       "step": 1,
-      "use_label": 18.0
+      "use_label": 0.0
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.976717112922002e-07,
-      "logits/chosen": -2.8282251358032227,
-      "logits/rejected": -2.8269264698028564,
-      "logps/chosen": -285.5703430175781,
-      "logps/rejected": -267.9831237792969,
-      "loss": 0.6805,
-      "pred_label": 479.7752380371094,
-      "rewards/accuracies": 0.5012626051902771,
-      "rewards/chosen": 0.0007524320390075445,
-      "rewards/margins": 0.0014539315598085523,
-      "rewards/rejected": -0.0007014995790086687,
+      "logits/chosen": -2.8195502758026123,
+      "logits/rejected": -2.828876495361328,
+      "logps/chosen": -285.2825012207031,
+      "logps/rejected": -270.9394836425781,
+      "loss": 0.6931,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.4873737394809723,
+      "rewards/chosen": -0.0001882972428575158,
+      "rewards/margins": -0.0012037099804729223,
+      "rewards/rejected": 0.0010154128540307283,
       "step": 100,
-      "use_label": 1138.2247314453125
+      "use_label": 0.0
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.3946449359720607e-07,
-      "logits/chosen": -2.829744338989258,
-      "logits/rejected": -2.8234996795654297,
-      "logps/chosen": -283.5379333496094,
-      "logps/rejected": -264.45965576171875,
-      "loss": 0.6797,
-      "pred_label": 1464.762451171875,
-      "rewards/accuracies": 0.5268750190734863,
-      "rewards/chosen": 0.0027550552040338516,
-      "rewards/margins": 0.004276593215763569,
-      "rewards/rejected": -0.0015215380117297173,
+      "logits/chosen": -2.8279786109924316,
+      "logits/rejected": -2.8164660930633545,
+      "logps/chosen": -278.72576904296875,
+      "logps/rejected": -260.5760498046875,
+      "loss": 0.6901,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.5506250262260437,
+      "rewards/chosen": 0.0056201983243227005,
+      "rewards/margins": 0.009352817200124264,
+      "rewards/rejected": -0.003732620272785425,
       "step": 200,
-      "use_label": 3337.237548828125
+      "use_label": 0.0
     },
     {
       "epoch": 0.31,
       "learning_rate": 3.812572759022118e-07,
-      "logits/chosen": -2.822391986846924,
-      "logits/rejected": -2.821011543273926,
-      "logps/chosen": -290.09552001953125,
-      "logps/rejected": -260.20050048828125,
-      "loss": 0.678,
-      "pred_label": 2522.33740234375,
-      "rewards/accuracies": 0.546875,
-      "rewards/chosen": 0.004526687320321798,
-      "rewards/margins": 0.008910334669053555,
-      "rewards/rejected": -0.004383646883070469,
+      "logits/chosen": -2.8140082359313965,
+      "logits/rejected": -2.8158328533172607,
+      "logps/chosen": -284.0040283203125,
+      "logps/rejected": -253.93580627441406,
+      "loss": 0.6875,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.5793750286102295,
+      "rewards/chosen": 0.006361996755003929,
+      "rewards/margins": 0.012689676135778427,
+      "rewards/rejected": -0.006327680312097073,
       "step": 300,
-      "use_label": 5479.66259765625
+      "use_label": 0.0
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.230500582072177e-07,
-      "logits/chosen": -2.8350696563720703,
-      "logits/rejected": -2.8237721920013428,
-      "logps/chosen": -284.8573303222656,
-      "logps/rejected": -260.8428039550781,
-      "loss": 0.6752,
-      "pred_label": 3636.47509765625,
-      "rewards/accuracies": 0.5778124928474426,
-      "rewards/chosen": 0.00710176769644022,
-      "rewards/margins": 0.013848603703081608,
-      "rewards/rejected": -0.006746836472302675,
+      "logits/chosen": -2.826840877532959,
+      "logits/rejected": -2.8091540336608887,
+      "logps/chosen": -284.3250427246094,
+      "logps/rejected": -262.8480529785156,
+      "loss": 0.6842,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": 0.010542460717260838,
+      "rewards/margins": 0.02265419438481331,
+      "rewards/rejected": -0.012111731804907322,
       "step": 400,
-      "use_label": 7565.52490234375
+      "use_label": 0.0
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.648428405122235e-07,
-      "logits/chosen": -2.8218374252319336,
-      "logits/rejected": -2.810873508453369,
-      "logps/chosen": -281.2003173828125,
-      "logps/rejected": -257.1551818847656,
-      "loss": 0.6707,
-      "pred_label": 4911.896484375,
-      "rewards/accuracies": 0.5731250047683716,
-      "rewards/chosen": 0.008172390051186085,
-      "rewards/margins": 0.017216255888342857,
-      "rewards/rejected": -0.009043867699801922,
+      "logits/chosen": -2.8074240684509277,
+      "logits/rejected": -2.795992136001587,
+      "logps/chosen": -269.97344970703125,
+      "logps/rejected": -252.08152770996094,
+      "loss": 0.6821,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.007951202802360058,
+      "rewards/margins": 0.017641538754105568,
+      "rewards/rejected": -0.00969033595174551,
       "step": 500,
-      "use_label": 9490.103515625
+      "use_label": 0.0
     },
     {
       "epoch": 0.63,
       "learning_rate": 2.0663562281722933e-07,
-      "logits/chosen": -2.8341524600982666,
-      "logits/rejected": -2.8230907917022705,
-      "logps/chosen": -284.7864685058594,
-      "logps/rejected": -262.0230712890625,
-      "loss": 0.6665,
-      "pred_label": 6349.55859375,
-      "rewards/accuracies": 0.6025000214576721,
-      "rewards/chosen": 0.012129506096243858,
-      "rewards/margins": 0.025582188740372658,
-      "rewards/rejected": -0.0134526826441288,
+      "logits/chosen": -2.8337247371673584,
+      "logits/rejected": -2.8203465938568115,
+      "logps/chosen": -285.0716857910156,
+      "logps/rejected": -265.4593811035156,
+      "loss": 0.6782,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.6356250047683716,
+      "rewards/chosen": 0.015922056511044502,
+      "rewards/margins": 0.035635706037282944,
+      "rewards/rejected": -0.01971365138888359,
       "step": 600,
-      "use_label": 11252.44140625
+      "use_label": 0.0
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.4842840512223514e-07,
-      "logits/chosen": -2.8274898529052734,
-      "logits/rejected": -2.811511516571045,
-      "logps/chosen": -282.0050048828125,
-      "logps/rejected": -252.4735565185547,
-      "loss": 0.6639,
-      "pred_label": 7854.15869140625,
-      "rewards/accuracies": 0.6112499833106995,
-      "rewards/chosen": 0.013030249625444412,
-      "rewards/margins": 0.026329634711146355,
-      "rewards/rejected": -0.013299385085701942,
+      "logits/chosen": -2.8270840644836426,
+      "logits/rejected": -2.8112306594848633,
+      "logps/chosen": -281.3733215332031,
+      "logps/rejected": -248.8465576171875,
+      "loss": 0.6762,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.6324999928474426,
+      "rewards/chosen": 0.018638433888554573,
+      "rewards/margins": 0.03718380257487297,
+      "rewards/rejected": -0.018545370548963547,
       "step": 700,
-      "use_label": 12947.8408203125
+      "use_label": 0.0
     },
     {
       "epoch": 0.84,
       "learning_rate": 9.022118742724097e-08,
-      "logits/chosen": -2.832928419113159,
-      "logits/rejected": -2.8318238258361816,
-      "logps/chosen": -285.62213134765625,
-      "logps/rejected": -259.8959045410156,
-      "loss": 0.662,
-      "pred_label": 9441.02734375,
-      "rewards/accuracies": 0.6253125071525574,
-      "rewards/chosen": 0.01489347591996193,
-      "rewards/margins": 0.030955424532294273,
-      "rewards/rejected": -0.016061950474977493,
+      "logits/chosen": -2.8297488689422607,
+      "logits/rejected": -2.83492374420166,
+      "logps/chosen": -288.54058837890625,
+      "logps/rejected": -261.6487731933594,
+      "loss": 0.6736,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.6393749713897705,
+      "rewards/chosen": 0.021646475419402122,
+      "rewards/margins": 0.04531754553318024,
+      "rewards/rejected": -0.023671068251132965,
       "step": 800,
-      "use_label": 14560.97265625
+      "use_label": 0.0
     },
     {
       "epoch": 0.94,
       "learning_rate": 3.20139697322468e-08,
-      "logits/chosen": -2.824517250061035,
-      "logits/rejected": -2.830662250518799,
-      "logps/chosen": -276.052490234375,
-      "logps/rejected": -262.5692138671875,
-      "loss": 0.6628,
-      "pred_label": 10994.255859375,
-      "rewards/accuracies": 0.6143749952316284,
-      "rewards/chosen": 0.012237527407705784,
-      "rewards/margins": 0.026263901963829994,
-      "rewards/rejected": -0.01402637455612421,
+      "logits/chosen": -2.8206725120544434,
+      "logits/rejected": -2.8274974822998047,
+      "logps/chosen": -277.310791015625,
+      "logps/rejected": -256.5542297363281,
+      "loss": 0.6744,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.6443750262260437,
+      "rewards/chosen": 0.019743308424949646,
+      "rewards/margins": 0.041582074016332626,
+      "rewards/rejected": -0.021838760003447533,
       "step": 900,
-      "use_label": 16207.744140625
+      "use_label": 0.0
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.841262102127075,
-      "eval_logits/rejected": -2.8343887329101562,
-      "eval_logps/chosen": -281.85919189453125,
-      "eval_logps/rejected": -262.4202880859375,
-      "eval_loss": 0.6618225574493408,
-      "eval_pred_label": 12855.98046875,
-      "eval_rewards/accuracies": 0.6150000095367432,
-      "eval_rewards/chosen": 0.011613711714744568,
-      "eval_rewards/margins": 0.02489962987601757,
-      "eval_rewards/rejected": -0.013285920023918152,
-      "eval_runtime": 826.7995,
-      "eval_samples_per_second": 2.419,
-      "eval_steps_per_second": 0.302,
-      "eval_use_label": 18206.01953125,
+      "eval_logits/chosen": -2.8419151306152344,
+      "eval_logits/rejected": -2.845423936843872,
+      "eval_logps/chosen": -284.0372314453125,
+      "eval_logps/rejected": -259.5419921875,
+      "eval_loss": 0.6741092801094055,
+      "eval_pred_label": 0.0,
+      "eval_rewards/accuracies": 0.6679999828338623,
+      "eval_rewards/chosen": 0.02201448194682598,
+      "eval_rewards/margins": 0.04624143987894058,
+      "eval_rewards/rejected": -0.02422695979475975,
+      "eval_runtime": 469.0597,
+      "eval_samples_per_second": 4.264,
+      "eval_steps_per_second": 0.266,
+      "eval_use_label": 0.0,
       "step": 955
     },
     {
       "epoch": 1.0,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.6705795382954063,
-      "train_runtime": 45840.6595,
-      "train_samples_per_second": 1.334,
-      "train_steps_per_second": 0.021
+      "train_loss": 0.6817296707193264,
+      "train_runtime": 25631.2708,
+      "train_samples_per_second": 2.385,
+      "train_steps_per_second": 0.037
     }
   ],
   "logging_steps": 100,
   "max_steps": 955,
   "num_train_epochs": 1,
-  "save_steps": 10,
+  "save_steps": 100,
   "total_flos": 0.0,
   "trial_name": null,
   "trial_params": null