{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9987943737441393,
  "eval_steps": 400,
  "global_step": 466,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010716677829872739,
      "grad_norm": 38.485095124933046,
      "learning_rate": 5.3191489361702123e-08,
      "logits/chosen": -2.398301601409912,
      "logits/rejected": -2.3920369148254395,
      "logps/chosen": -0.5667670965194702,
      "logps/rejected": -0.5555010437965393,
      "loss": 1.5468,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5667670965194702,
      "rewards/margins": -0.011266030371189117,
      "rewards/rejected": -0.5555010437965393,
      "step": 5
    },
    {
      "epoch": 0.021433355659745478,
      "grad_norm": 18.322790262213086,
      "learning_rate": 1.0638297872340425e-07,
      "logits/chosen": -2.4026951789855957,
      "logits/rejected": -2.3731987476348877,
      "logps/chosen": -0.5517200231552124,
      "logps/rejected": -0.5781593918800354,
      "loss": 1.554,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5517200231552124,
      "rewards/margins": 0.026439335197210312,
      "rewards/rejected": -0.5781593918800354,
      "step": 10
    },
    {
      "epoch": 0.032150033489618215,
      "grad_norm": 17.062192600734857,
      "learning_rate": 1.5957446808510638e-07,
      "logits/chosen": -2.4439353942871094,
      "logits/rejected": -2.449899196624756,
      "logps/chosen": -0.5637143850326538,
      "logps/rejected": -0.567409336566925,
      "loss": 1.5617,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5637143850326538,
      "rewards/margins": 0.0036949075292795897,
      "rewards/rejected": -0.567409336566925,
      "step": 15
    },
    {
      "epoch": 0.042866711319490956,
      "grad_norm": 17.524849197474367,
      "learning_rate": 2.127659574468085e-07,
      "logits/chosen": -2.384251117706299,
      "logits/rejected": -2.3945975303649902,
      "logps/chosen": -0.5456732511520386,
      "logps/rejected": -0.5427749752998352,
      "loss": 1.5321,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5456732511520386,
      "rewards/margins": -0.002898297505453229,
      "rewards/rejected": -0.5427749752998352,
      "step": 20
    },
    {
      "epoch": 0.0535833891493637,
      "grad_norm": 14.321312880487431,
      "learning_rate": 2.659574468085106e-07,
      "logits/chosen": -2.279252529144287,
      "logits/rejected": -2.2811872959136963,
      "logps/chosen": -0.5260757803916931,
      "logps/rejected": -0.543364405632019,
      "loss": 1.53,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5260757803916931,
      "rewards/margins": 0.017288658767938614,
      "rewards/rejected": -0.543364405632019,
      "step": 25
    },
    {
      "epoch": 0.06430006697923643,
      "grad_norm": 19.544853384077058,
      "learning_rate": 3.1914893617021275e-07,
      "logits/chosen": -2.3907198905944824,
      "logits/rejected": -2.403733015060425,
      "logps/chosen": -0.5487326979637146,
      "logps/rejected": -0.5386630892753601,
      "loss": 1.5515,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.5487326979637146,
      "rewards/margins": -0.010069593787193298,
      "rewards/rejected": -0.5386630892753601,
      "step": 30
    },
    {
      "epoch": 0.07501674480910918,
      "grad_norm": 24.103155081211728,
      "learning_rate": 3.7234042553191484e-07,
      "logits/chosen": -2.452573299407959,
      "logits/rejected": -2.423945665359497,
      "logps/chosen": -0.5365626811981201,
      "logps/rejected": -0.5920503735542297,
      "loss": 1.5478,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.5365626811981201,
      "rewards/margins": 0.055487751960754395,
      "rewards/rejected": -0.5920503735542297,
      "step": 35
    },
    {
      "epoch": 0.08573342263898191,
      "grad_norm": 21.06293839287325,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -2.3676323890686035,
      "logits/rejected": -2.349027156829834,
      "logps/chosen": -0.5672295093536377,
      "logps/rejected": -0.5381959676742554,
      "loss": 1.5581,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.5672295093536377,
      "rewards/margins": -0.02903353050351143,
      "rewards/rejected": -0.5381959676742554,
      "step": 40
    },
    {
      "epoch": 0.09645010046885466,
      "grad_norm": 17.138121888291746,
      "learning_rate": 4.787234042553192e-07,
      "logits/chosen": -2.334918260574341,
      "logits/rejected": -2.3297195434570312,
      "logps/chosen": -0.5420322418212891,
      "logps/rejected": -0.560600757598877,
      "loss": 1.5426,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5420322418212891,
      "rewards/margins": 0.01856859028339386,
      "rewards/rejected": -0.560600757598877,
      "step": 45
    },
    {
      "epoch": 0.1071667782987274,
      "grad_norm": 16.650052527651823,
      "learning_rate": 4.999367579541646e-07,
      "logits/chosen": -2.3611702919006348,
      "logits/rejected": -2.3566925525665283,
      "logps/chosen": -0.542779803276062,
      "logps/rejected": -0.5264804363250732,
      "loss": 1.5287,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.542779803276062,
      "rewards/margins": -0.016299355775117874,
      "rewards/rejected": -0.5264804363250732,
      "step": 50
    },
    {
      "epoch": 0.11788345612860013,
      "grad_norm": 25.231872930806986,
      "learning_rate": 4.99550394651012e-07,
      "logits/chosen": -2.4198124408721924,
      "logits/rejected": -2.4228649139404297,
      "logps/chosen": -0.5739614367485046,
      "logps/rejected": -0.5911067128181458,
      "loss": 1.5407,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.5739614367485046,
      "rewards/margins": 0.017145240679383278,
      "rewards/rejected": -0.5911067128181458,
      "step": 55
    },
    {
      "epoch": 0.12860013395847286,
      "grad_norm": 27.654893603033948,
      "learning_rate": 4.988133448023071e-07,
      "logits/chosen": -2.3333356380462646,
      "logits/rejected": -2.342374324798584,
      "logps/chosen": -0.505652129650116,
      "logps/rejected": -0.52570641040802,
      "loss": 1.5384,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.505652129650116,
      "rewards/margins": 0.020054226741194725,
      "rewards/rejected": -0.52570641040802,
      "step": 60
    },
    {
      "epoch": 0.13931681178834562,
      "grad_norm": 18.301707545724692,
      "learning_rate": 4.977266441646379e-07,
      "logits/chosen": -2.3869385719299316,
      "logits/rejected": -2.391303777694702,
      "logps/chosen": -0.5198084712028503,
      "logps/rejected": -0.5502195358276367,
      "loss": 1.5466,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5198084712028503,
      "rewards/margins": 0.030411023646593094,
      "rewards/rejected": -0.5502195358276367,
      "step": 65
    },
    {
      "epoch": 0.15003348961821836,
      "grad_norm": 19.461976097246225,
      "learning_rate": 4.962918198495654e-07,
      "logits/chosen": -2.2925384044647217,
      "logits/rejected": -2.296604633331299,
      "logps/chosen": -0.5172106027603149,
      "logps/rejected": -0.5665396451950073,
      "loss": 1.5322,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5172106027603149,
      "rewards/margins": 0.04932904243469238,
      "rewards/rejected": -0.5665396451950073,
      "step": 70
    },
    {
      "epoch": 0.1607501674480911,
      "grad_norm": 16.227392616134246,
      "learning_rate": 4.945108881776135e-07,
      "logits/chosen": -2.3317081928253174,
      "logits/rejected": -2.3258421421051025,
      "logps/chosen": -0.48806238174438477,
      "logps/rejected": -0.5191291570663452,
      "loss": 1.5323,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.48806238174438477,
      "rewards/margins": 0.031066760420799255,
      "rewards/rejected": -0.5191291570663452,
      "step": 75
    },
    {
      "epoch": 0.17146684527796383,
      "grad_norm": 18.668922205376074,
      "learning_rate": 4.923863518447879e-07,
      "logits/chosen": -2.304633617401123,
      "logits/rejected": -2.3267812728881836,
      "logps/chosen": -0.4988403916358948,
      "logps/rejected": -0.5690027475357056,
      "loss": 1.5304,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.4988403916358948,
      "rewards/margins": 0.07016237080097198,
      "rewards/rejected": -0.5690027475357056,
      "step": 80
    },
    {
      "epoch": 0.18218352310783656,
      "grad_norm": 21.78609949315303,
      "learning_rate": 4.899211964056035e-07,
      "logits/chosen": -2.338728427886963,
      "logits/rejected": -2.331895112991333,
      "logps/chosen": -0.5135633945465088,
      "logps/rejected": -0.5660439729690552,
      "loss": 1.5365,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5135633945465088,
      "rewards/margins": 0.052480556070804596,
      "rewards/rejected": -0.5660439729690552,
      "step": 85
    },
    {
      "epoch": 0.19290020093770932,
      "grad_norm": 18.81785585703593,
      "learning_rate": 4.871188860775643e-07,
      "logits/chosen": -2.288607120513916,
      "logits/rejected": -2.292832374572754,
      "logps/chosen": -0.5013225674629211,
      "logps/rejected": -0.5400410890579224,
      "loss": 1.5369,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.5013225674629211,
      "rewards/margins": 0.03871860355138779,
      "rewards/rejected": -0.5400410890579224,
      "step": 90
    },
    {
      "epoch": 0.20361687876758205,
      "grad_norm": 18.284986731616815,
      "learning_rate": 4.839833588729897e-07,
      "logits/chosen": -2.33103609085083,
      "logits/rejected": -2.3427205085754395,
      "logps/chosen": -0.5798801779747009,
      "logps/rejected": -0.5994023680686951,
      "loss": 1.5235,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.5798801779747009,
      "rewards/margins": 0.01952212117612362,
      "rewards/rejected": -0.5994023680686951,
      "step": 95
    },
    {
      "epoch": 0.2143335565974548,
      "grad_norm": 24.996484906718788,
      "learning_rate": 4.80519021065031e-07,
      "logits/chosen": -2.3483448028564453,
      "logits/rejected": -2.3573968410491943,
      "logps/chosen": -0.5851219296455383,
      "logps/rejected": -0.6033455729484558,
      "loss": 1.5464,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.5851219296455383,
      "rewards/margins": 0.018223673105239868,
      "rewards/rejected": -0.6033455729484558,
      "step": 100
    },
    {
      "epoch": 0.22505023442732752,
      "grad_norm": 15.841869984614812,
      "learning_rate": 4.767307409956528e-07,
      "logits/chosen": -2.3424832820892334,
      "logits/rejected": -2.342660903930664,
      "logps/chosen": -0.5200175046920776,
      "logps/rejected": -0.5671975612640381,
      "loss": 1.5177,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.5200175046920776,
      "rewards/margins": 0.04718000069260597,
      "rewards/rejected": -0.5671975612640381,
      "step": 105
    },
    {
      "epoch": 0.23576691225720026,
      "grad_norm": 26.162342495475848,
      "learning_rate": 4.726238422342805e-07,
      "logits/chosen": -2.370981216430664,
      "logits/rejected": -2.377699375152588,
      "logps/chosen": -0.5119336247444153,
      "logps/rejected": -0.5585616230964661,
      "loss": 1.5421,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5119336247444153,
      "rewards/margins": 0.04662800580263138,
      "rewards/rejected": -0.5585616230964661,
      "step": 110
    },
    {
      "epoch": 0.24648359008707302,
      "grad_norm": 22.223934331525054,
      "learning_rate": 4.6820409609673027e-07,
      "logits/chosen": -2.445955276489258,
      "logits/rejected": -2.439049005508423,
      "logps/chosen": -0.507788360118866,
      "logps/rejected": -0.5285546779632568,
      "loss": 1.573,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.507788360118866,
      "rewards/margins": 0.02076634392142296,
      "rewards/rejected": -0.5285546779632568,
      "step": 115
    },
    {
      "epoch": 0.2572002679169457,
      "grad_norm": 17.337249756046997,
      "learning_rate": 4.6347771353493173e-07,
      "logits/chosen": -2.507343053817749,
      "logits/rejected": -2.4980244636535645,
      "logps/chosen": -0.5571922063827515,
      "logps/rejected": -0.5488871335983276,
      "loss": 1.5546,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.5571922063827515,
      "rewards/margins": -0.008305097930133343,
      "rewards/rejected": -0.5488871335983276,
      "step": 120
    },
    {
      "epoch": 0.2679169457468185,
      "grad_norm": 18.61479199807285,
      "learning_rate": 4.5845133640884217e-07,
      "logits/chosen": -2.496476173400879,
      "logits/rejected": -2.5013513565063477,
      "logps/chosen": -0.5598434209823608,
      "logps/rejected": -0.5717306733131409,
      "loss": 1.5472,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5598434209823608,
      "rewards/margins": 0.01188725233078003,
      "rewards/rejected": -0.5717306733131409,
      "step": 125
    },
    {
      "epoch": 0.27863362357669125,
      "grad_norm": 24.242840152077708,
      "learning_rate": 4.531320281528169e-07,
      "logits/chosen": -2.577277660369873,
      "logits/rejected": -2.5614867210388184,
      "logps/chosen": -0.5257032513618469,
      "logps/rejected": -0.5607699155807495,
      "loss": 1.5206,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5257032513618469,
      "rewards/margins": 0.03506666049361229,
      "rewards/rejected": -0.5607699155807495,
      "step": 130
    },
    {
      "epoch": 0.289350301406564,
      "grad_norm": 28.052518647744463,
      "learning_rate": 4.4752726384955293e-07,
      "logits/chosen": -2.5903048515319824,
      "logits/rejected": -2.6012890338897705,
      "logps/chosen": -0.5207422971725464,
      "logps/rejected": -0.5546741485595703,
      "loss": 1.5394,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5207422971725464,
      "rewards/margins": 0.03393186628818512,
      "rewards/rejected": -0.5546741485595703,
      "step": 135
    },
    {
      "epoch": 0.3000669792364367,
      "grad_norm": 20.236382854852724,
      "learning_rate": 4.4164491972555294e-07,
      "logits/chosen": -2.6124188899993896,
      "logits/rejected": -2.5999808311462402,
      "logps/chosen": -0.5296235084533691,
      "logps/rejected": -0.5590516924858093,
      "loss": 1.5202,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5296235084533691,
      "rewards/margins": 0.02942819520831108,
      "rewards/rejected": -0.5590516924858093,
      "step": 140
    },
    {
      "epoch": 0.31078365706630945,
      "grad_norm": 18.380103954877008,
      "learning_rate": 4.3549326208287393e-07,
      "logits/chosen": -2.710564613342285,
      "logits/rejected": -2.68730092048645,
      "logps/chosen": -0.5258200764656067,
      "logps/rejected": -0.5745820999145508,
      "loss": 1.5158,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.5258200764656067,
      "rewards/margins": 0.0487620048224926,
      "rewards/rejected": -0.5745820999145508,
      "step": 145
    },
    {
      "epoch": 0.3215003348961822,
      "grad_norm": 19.20157418312748,
      "learning_rate": 4.2908093568271187e-07,
      "logits/chosen": -2.70224666595459,
      "logits/rejected": -2.694885730743408,
      "logps/chosen": -0.5079634189605713,
      "logps/rejected": -0.5393082499504089,
      "loss": 1.5455,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5079634189605713,
      "rewards/margins": 0.03134483844041824,
      "rewards/rejected": -0.5393082499504089,
      "step": 150
    },
    {
      "epoch": 0.3322170127260549,
      "grad_norm": 17.41360540258233,
      "learning_rate": 4.2241695159714843e-07,
      "logits/chosen": -2.793240785598755,
      "logits/rejected": -2.801339626312256,
      "logps/chosen": -0.5376655459403992,
      "logps/rejected": -0.5706537365913391,
      "loss": 1.5278,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5376655459403992,
      "rewards/margins": 0.03298817574977875,
      "rewards/rejected": -0.5706537365913391,
      "step": 155
    },
    {
      "epoch": 0.34293369055592765,
      "grad_norm": 21.649135642065247,
      "learning_rate": 4.1551067454613074e-07,
      "logits/chosen": -2.802827835083008,
      "logits/rejected": -2.78732967376709,
      "logps/chosen": -0.558406412601471,
      "logps/rejected": -0.6224438548088074,
      "loss": 1.5318,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.558406412601471,
      "rewards/margins": 0.06403747946023941,
      "rewards/rejected": -0.6224438548088074,
      "step": 160
    },
    {
      "epoch": 0.3536503683858004,
      "grad_norm": 20.48435877400994,
      "learning_rate": 4.0837180973747876e-07,
      "logits/chosen": -2.9101483821868896,
      "logits/rejected": -2.8818156719207764,
      "logps/chosen": -0.5387953519821167,
      "logps/rejected": -0.571646511554718,
      "loss": 1.5233,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.5387953519821167,
      "rewards/margins": 0.032851073890924454,
      "rewards/rejected": -0.571646511554718,
      "step": 165
    },
    {
      "epoch": 0.3643670462156731,
      "grad_norm": 17.62876339336723,
      "learning_rate": 4.0101038922841455e-07,
      "logits/chosen": -2.9059550762176514,
      "logits/rejected": -2.9104220867156982,
      "logps/chosen": -0.5236160159111023,
      "logps/rejected": -0.5999457836151123,
      "loss": 1.5112,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5236160159111023,
      "rewards/margins": 0.07632973045110703,
      "rewards/rejected": -0.5999457836151123,
      "step": 170
    },
    {
      "epoch": 0.3750837240455459,
      "grad_norm": 20.453674701254528,
      "learning_rate": 3.934367578277783e-07,
      "logits/chosen": -2.9715123176574707,
      "logits/rejected": -2.953988790512085,
      "logps/chosen": -0.5574652552604675,
      "logps/rejected": -0.5864735841751099,
      "loss": 1.5348,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5574652552604675,
      "rewards/margins": 0.029008299112319946,
      "rewards/rejected": -0.5864735841751099,
      "step": 175
    },
    {
      "epoch": 0.38580040187541864,
      "grad_norm": 22.621519510630307,
      "learning_rate": 3.856615585587434e-07,
      "logits/chosen": -3.067772388458252,
      "logits/rejected": -3.0693717002868652,
      "logps/chosen": -0.5790558457374573,
      "logps/rejected": -0.6112508177757263,
      "loss": 1.545,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5790558457374573,
      "rewards/margins": 0.03219496086239815,
      "rewards/rejected": -0.6112508177757263,
      "step": 180
    },
    {
      "epoch": 0.3965170797052914,
      "grad_norm": 27.765398609724794,
      "learning_rate": 3.776957177024581e-07,
      "logits/chosen": -3.070247173309326,
      "logits/rejected": -3.0919833183288574,
      "logps/chosen": -0.5750337839126587,
      "logps/rejected": -0.6032044887542725,
      "loss": 1.5218,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.5750337839126587,
      "rewards/margins": 0.028170805424451828,
      "rewards/rejected": -0.6032044887542725,
      "step": 185
    },
    {
      "epoch": 0.4072337575351641,
      "grad_norm": 19.099968753214323,
      "learning_rate": 3.69550429443633e-07,
      "logits/chosen": -2.944047451019287,
      "logits/rejected": -2.959052324295044,
      "logps/chosen": -0.5526596307754517,
      "logps/rejected": -0.7156730890274048,
      "loss": 1.5268,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5526596307754517,
      "rewards/margins": 0.16301342844963074,
      "rewards/rejected": -0.7156730890274048,
      "step": 190
    },
    {
      "epoch": 0.41795043536503684,
      "grad_norm": 24.556600066756868,
      "learning_rate": 3.612371401396502e-07,
      "logits/chosen": -3.195617198944092,
      "logits/rejected": -3.168480396270752,
      "logps/chosen": -0.515397846698761,
      "logps/rejected": -0.5484700798988342,
      "loss": 1.5305,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.515397846698761,
      "rewards/margins": 0.03307227045297623,
      "rewards/rejected": -0.5484700798988342,
      "step": 195
    },
    {
      "epoch": 0.4286671131949096,
      "grad_norm": 20.445564921711252,
      "learning_rate": 3.527675322353011e-07,
      "logits/chosen": -3.071345567703247,
      "logits/rejected": -3.0912113189697266,
      "logps/chosen": -0.4999586045742035,
      "logps/rejected": -0.5286678075790405,
      "loss": 1.5201,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.4999586045742035,
      "rewards/margins": 0.028709203004837036,
      "rewards/rejected": -0.5286678075790405,
      "step": 200
    },
    {
      "epoch": 0.4393837910247823,
      "grad_norm": 28.017877985161014,
      "learning_rate": 3.4415350784575693e-07,
      "logits/chosen": -3.2119345664978027,
      "logits/rejected": -3.2098147869110107,
      "logps/chosen": -0.48476672172546387,
      "logps/rejected": -0.5175997018814087,
      "loss": 1.5335,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.48476672172546387,
      "rewards/margins": 0.032833002507686615,
      "rewards/rejected": -0.5175997018814087,
      "step": 205
    },
    {
      "epoch": 0.45010046885465504,
      "grad_norm": 20.446366205479023,
      "learning_rate": 3.3540717203084224e-07,
      "logits/chosen": -3.2167842388153076,
      "logits/rejected": -3.209538698196411,
      "logps/chosen": -0.5124386548995972,
      "logps/rejected": -0.5561140179634094,
      "loss": 1.5035,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5124386548995972,
      "rewards/margins": 0.04367532581090927,
      "rewards/rejected": -0.5561140179634094,
      "step": 210
    },
    {
      "epoch": 0.4608171466845278,
      "grad_norm": 20.249310261680897,
      "learning_rate": 3.265408157841153e-07,
      "logits/chosen": -3.2260100841522217,
      "logits/rejected": -3.2222793102264404,
      "logps/chosen": -0.5612623691558838,
      "logps/rejected": -0.6017634272575378,
      "loss": 1.4952,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.5612623691558838,
      "rewards/margins": 0.040501099079847336,
      "rewards/rejected": -0.6017634272575378,
      "step": 215
    },
    {
      "epoch": 0.4715338245144005,
      "grad_norm": 28.781450630078922,
      "learning_rate": 3.1756689876066187e-07,
      "logits/chosen": -3.26216459274292,
      "logits/rejected": -3.2409801483154297,
      "logps/chosen": -0.5308222770690918,
      "logps/rejected": -0.609308660030365,
      "loss": 1.5377,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.5308222770690918,
      "rewards/margins": 0.078486368060112,
      "rewards/rejected": -0.609308660030365,
      "step": 220
    },
    {
      "epoch": 0.4822505023442733,
      "grad_norm": 20.9210927889319,
      "learning_rate": 3.0849803176787184e-07,
      "logits/chosen": -3.3171279430389404,
      "logits/rejected": -3.3153088092803955,
      "logps/chosen": -0.5533649921417236,
      "logps/rejected": -0.6165200471878052,
      "loss": 1.5082,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5533649921417236,
      "rewards/margins": 0.06315504014492035,
      "rewards/rejected": -0.6165200471878052,
      "step": 225
    },
    {
      "epoch": 0.49296718017414604,
      "grad_norm": 26.953714991186636,
      "learning_rate": 2.993469590438065e-07,
      "logits/chosen": -3.2146403789520264,
      "logits/rejected": -3.2017455101013184,
      "logps/chosen": -0.539149284362793,
      "logps/rejected": -0.5770600438117981,
      "loss": 1.4968,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.539149284362793,
      "rewards/margins": 0.03791072219610214,
      "rewards/rejected": -0.5770600438117981,
      "step": 230
    },
    {
      "epoch": 0.5036838580040187,
      "grad_norm": 23.063142701753478,
      "learning_rate": 2.901265403480597e-07,
      "logits/chosen": -3.2051620483398438,
      "logits/rejected": -3.1935036182403564,
      "logps/chosen": -0.5151172876358032,
      "logps/rejected": -0.6125957369804382,
      "loss": 1.508,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.5151172876358032,
      "rewards/margins": 0.09747843444347382,
      "rewards/rejected": -0.6125957369804382,
      "step": 235
    },
    {
      "epoch": 0.5144005358338914,
      "grad_norm": 28.679041181758652,
      "learning_rate": 2.808497328902782e-07,
      "logits/chosen": -3.152073621749878,
      "logits/rejected": -3.15959095954895,
      "logps/chosen": -0.5507190823554993,
      "logps/rejected": -0.6257890462875366,
      "loss": 1.5193,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5507190823554993,
      "rewards/margins": 0.07506996393203735,
      "rewards/rejected": -0.6257890462875366,
      "step": 240
    },
    {
      "epoch": 0.5251172136637642,
      "grad_norm": 24.21410810508721,
      "learning_rate": 2.715295731217396e-07,
      "logits/chosen": -3.1899075508117676,
      "logits/rejected": -3.1687207221984863,
      "logps/chosen": -0.5879708528518677,
      "logps/rejected": -0.6203640103340149,
      "loss": 1.5033,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.5879708528518677,
      "rewards/margins": 0.032393116503953934,
      "rewards/rejected": -0.6203640103340149,
      "step": 245
    },
    {
      "epoch": 0.535833891493637,
      "grad_norm": 24.944015170343583,
      "learning_rate": 2.621791584155725e-07,
      "logits/chosen": -3.186570167541504,
      "logits/rejected": -3.1771557331085205,
      "logps/chosen": -0.610142171382904,
      "logps/rejected": -0.604741096496582,
      "loss": 1.5341,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.610142171382904,
      "rewards/margins": -0.005401057191193104,
      "rewards/rejected": -0.604741096496582,
      "step": 250
    },
    {
      "epoch": 0.5465505693235098,
      "grad_norm": 22.133595287604955,
      "learning_rate": 2.52811628661367e-07,
      "logits/chosen": -3.2132976055145264,
      "logits/rejected": -3.21321439743042,
      "logps/chosen": -0.5207762122154236,
      "logps/rejected": -0.5928905606269836,
      "loss": 1.4997,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5207762122154236,
      "rewards/margins": 0.07211437076330185,
      "rewards/rejected": -0.5928905606269836,
      "step": 255
    },
    {
      "epoch": 0.5572672471533825,
      "grad_norm": 21.31026102730006,
      "learning_rate": 2.434401478000353e-07,
      "logits/chosen": -3.141242504119873,
      "logits/rejected": -3.1591105461120605,
      "logps/chosen": -0.526002049446106,
      "logps/rejected": -0.6240478754043579,
      "loss": 1.5259,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.526002049446106,
      "rewards/margins": 0.09804578125476837,
      "rewards/rejected": -0.6240478754043579,
      "step": 260
    },
    {
      "epoch": 0.5679839249832552,
      "grad_norm": 25.465212058173062,
      "learning_rate": 2.340778853248759e-07,
      "logits/chosen": -3.1647770404815674,
      "logits/rejected": -3.161611557006836,
      "logps/chosen": -0.5447332859039307,
      "logps/rejected": -0.6431838870048523,
      "loss": 1.496,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5447332859039307,
      "rewards/margins": 0.098450668156147,
      "rewards/rejected": -0.6431838870048523,
      "step": 265
    },
    {
      "epoch": 0.578700602813128,
      "grad_norm": 29.79126766513931,
      "learning_rate": 2.247379977748339e-07,
      "logits/chosen": -3.1814353466033936,
      "logits/rejected": -3.1744346618652344,
      "logps/chosen": -0.5134252905845642,
      "logps/rejected": -0.5546069145202637,
      "loss": 1.5172,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.5134252905845642,
      "rewards/margins": 0.04118159040808678,
      "rewards/rejected": -0.5546069145202637,
      "step": 270
    },
    {
      "epoch": 0.5894172806430007,
      "grad_norm": 20.911132725985905,
      "learning_rate": 2.1543361024596573e-07,
      "logits/chosen": -3.2813096046447754,
      "logits/rejected": -3.2904067039489746,
      "logps/chosen": -0.478159099817276,
      "logps/rejected": -0.5237951278686523,
      "loss": 1.52,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.478159099817276,
      "rewards/margins": 0.045636050403118134,
      "rewards/rejected": -0.5237951278686523,
      "step": 275
    },
    {
      "epoch": 0.6001339584728734,
      "grad_norm": 22.184303638777845,
      "learning_rate": 2.0617779794709082e-07,
      "logits/chosen": -3.227949857711792,
      "logits/rejected": -3.199709415435791,
      "logps/chosen": -0.5574294924736023,
      "logps/rejected": -0.6131659746170044,
      "loss": 1.5077,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5574294924736023,
      "rewards/margins": 0.05573650076985359,
      "rewards/rejected": -0.6131659746170044,
      "step": 280
    },
    {
      "epoch": 0.6108506363027462,
      "grad_norm": 23.16627725406288,
      "learning_rate": 1.9698356782554687e-07,
      "logits/chosen": -3.2686588764190674,
      "logits/rejected": -3.27409291267395,
      "logps/chosen": -0.6056652069091797,
      "logps/rejected": -0.6142518520355225,
      "loss": 1.529,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.6056652069091797,
      "rewards/margins": 0.008586685173213482,
      "rewards/rejected": -0.6142518520355225,
      "step": 285
    },
    {
      "epoch": 0.6215673141326189,
      "grad_norm": 43.63471821872903,
      "learning_rate": 1.878638402888727e-07,
      "logits/chosen": -3.3371078968048096,
      "logits/rejected": -3.3204472064971924,
      "logps/chosen": -0.5225822925567627,
      "logps/rejected": -0.5866876840591431,
      "loss": 1.5024,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.5225822925567627,
      "rewards/margins": 0.06410545110702515,
      "rewards/rejected": -0.5866876840591431,
      "step": 290
    },
    {
      "epoch": 0.6322839919624916,
      "grad_norm": 25.701019654379248,
      "learning_rate": 1.7883143104810223e-07,
      "logits/chosen": -3.2457165718078613,
      "logits/rejected": -3.2686493396759033,
      "logps/chosen": -0.5675607919692993,
      "logps/rejected": -0.6349104642868042,
      "loss": 1.4993,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.5675607919692993,
      "rewards/margins": 0.06734969466924667,
      "rewards/rejected": -0.6349104642868042,
      "step": 295
    },
    {
      "epoch": 0.6430006697923644,
      "grad_norm": 23.619470082480703,
      "learning_rate": 1.6989903310818548e-07,
      "logits/chosen": -3.3820648193359375,
      "logits/rejected": -3.3857345581054688,
      "logps/chosen": -0.5343354344367981,
      "logps/rejected": -0.6022351980209351,
      "loss": 1.4951,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.5343354344367981,
      "rewards/margins": 0.06789974123239517,
      "rewards/rejected": -0.6022351980209351,
      "step": 300
    },
    {
      "epoch": 0.6537173476222371,
      "grad_norm": 22.753889982587108,
      "learning_rate": 1.610791989308466e-07,
      "logits/chosen": -3.3234715461730957,
      "logits/rejected": -3.2950637340545654,
      "logps/chosen": -0.5346772074699402,
      "logps/rejected": -0.5783089995384216,
      "loss": 1.5037,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5346772074699402,
      "rewards/margins": 0.04363187402486801,
      "rewards/rejected": -0.5783089995384216,
      "step": 305
    },
    {
      "epoch": 0.6644340254521098,
      "grad_norm": 29.970511411150166,
      "learning_rate": 1.523843227949418e-07,
      "logits/chosen": -3.3544983863830566,
      "logits/rejected": -3.355292558670044,
      "logps/chosen": -0.5303682088851929,
      "logps/rejected": -0.5744400024414062,
      "loss": 1.4936,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.5303682088851929,
      "rewards/margins": 0.04407178610563278,
      "rewards/rejected": -0.5744400024414062,
      "step": 310
    },
    {
      "epoch": 0.6751507032819826,
      "grad_norm": 19.95585897697304,
      "learning_rate": 1.4382662337910867e-07,
      "logits/chosen": -3.291701078414917,
      "logits/rejected": -3.3025760650634766,
      "logps/chosen": -0.5352946519851685,
      "logps/rejected": -0.5887657403945923,
      "loss": 1.502,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.5352946519851685,
      "rewards/margins": 0.053471069782972336,
      "rewards/rejected": -0.5887657403945923,
      "step": 315
    },
    {
      "epoch": 0.6858673811118553,
      "grad_norm": 22.149800280030288,
      "learning_rate": 1.3541812659118104e-07,
      "logits/chosen": -3.334624767303467,
      "logits/rejected": -3.306361675262451,
      "logps/chosen": -0.5542501211166382,
      "logps/rejected": -0.5587230920791626,
      "loss": 1.524,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5542501211166382,
      "rewards/margins": 0.004472990520298481,
      "rewards/rejected": -0.5587230920791626,
      "step": 320
    },
    {
      "epoch": 0.696584058941728,
      "grad_norm": 25.604347826249015,
      "learning_rate": 1.2717064866850047e-07,
      "logits/chosen": -3.3316922187805176,
      "logits/rejected": -3.342952251434326,
      "logps/chosen": -0.5008823275566101,
      "logps/rejected": -0.5843838453292847,
      "loss": 1.4797,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5008823275566101,
      "rewards/margins": 0.08350151777267456,
      "rewards/rejected": -0.5843838453292847,
      "step": 325
    },
    {
      "epoch": 0.7073007367716008,
      "grad_norm": 24.090724072052794,
      "learning_rate": 1.1909577957287117e-07,
      "logits/chosen": -3.3231868743896484,
      "logits/rejected": -3.2961513996124268,
      "logps/chosen": -0.5285135507583618,
      "logps/rejected": -0.582491397857666,
      "loss": 1.4972,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5285135507583618,
      "rewards/margins": 0.05397792533040047,
      "rewards/rejected": -0.582491397857666,
      "step": 330
    },
    {
      "epoch": 0.7180174146014735,
      "grad_norm": 36.87118130194036,
      "learning_rate": 1.1120486670349441e-07,
      "logits/chosen": -3.4078221321105957,
      "logits/rejected": -3.3980777263641357,
      "logps/chosen": -0.5307896733283997,
      "logps/rejected": -0.6150525808334351,
      "loss": 1.5019,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.5307896733283997,
      "rewards/margins": 0.08426298201084137,
      "rewards/rejected": -0.6150525808334351,
      "step": 335
    },
    {
      "epoch": 0.7287340924313462,
      "grad_norm": 25.52897276938915,
      "learning_rate": 1.0350899895076948e-07,
      "logits/chosen": -3.3364510536193848,
      "logits/rejected": -3.321132183074951,
      "logps/chosen": -0.5735017657279968,
      "logps/rejected": -0.6429983973503113,
      "loss": 1.521,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.5735017657279968,
      "rewards/margins": 0.06949658691883087,
      "rewards/rejected": -0.6429983973503113,
      "step": 340
    },
    {
      "epoch": 0.739450770261219,
      "grad_norm": 20.331652657630677,
      "learning_rate": 9.601899111337069e-08,
      "logits/chosen": -3.367867946624756,
      "logits/rejected": -3.3775951862335205,
      "logps/chosen": -0.5771843194961548,
      "logps/rejected": -0.6704375147819519,
      "loss": 1.496,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.5771843194961548,
      "rewards/margins": 0.09325330704450607,
      "rewards/rejected": -0.6704375147819519,
      "step": 345
    },
    {
      "epoch": 0.7501674480910918,
      "grad_norm": 22.274114959082404,
      "learning_rate": 8.874536870049765e-08,
      "logits/chosen": -3.3537139892578125,
      "logits/rejected": -3.378490924835205,
      "logps/chosen": -0.5365322828292847,
      "logps/rejected": -0.6033391952514648,
      "loss": 1.4971,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.5365322828292847,
      "rewards/margins": 0.06680694222450256,
      "rewards/rejected": -0.6033391952514648,
      "step": 350
    },
    {
      "epoch": 0.7608841259209645,
      "grad_norm": 24.177285206790465,
      "learning_rate": 8.169835314065663e-08,
      "logits/chosen": -3.3511605262756348,
      "logits/rejected": -3.3401741981506348,
      "logps/chosen": -0.5807870626449585,
      "logps/rejected": -0.6009158492088318,
      "loss": 1.499,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.5807870626449585,
      "rewards/margins": 0.020128777250647545,
      "rewards/rejected": -0.6009158492088318,
      "step": 355
    },
    {
      "epoch": 0.7716008037508373,
      "grad_norm": 22.05484832098868,
      "learning_rate": 7.488784741775816e-08,
      "logits/chosen": -3.3435721397399902,
      "logits/rejected": -3.33024525642395,
      "logps/chosen": -0.5081290006637573,
      "logps/rejected": -0.5656172037124634,
      "loss": 1.5079,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5081290006637573,
      "rewards/margins": 0.057488154619932175,
      "rewards/rejected": -0.5656172037124634,
      "step": 360
    },
    {
      "epoch": 0.78231748158071,
      "grad_norm": 24.600234649150252,
      "learning_rate": 6.832342215471715e-08,
      "logits/chosen": -3.3981361389160156,
      "logits/rejected": -3.39165997505188,
      "logps/chosen": -0.5507721304893494,
      "logps/rejected": -0.6153653860092163,
      "loss": 1.5147,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5507721304893494,
      "rewards/margins": 0.06459329277276993,
      "rewards/rejected": -0.6153653860092163,
      "step": 365
    },
    {
      "epoch": 0.7930341594105828,
      "grad_norm": 31.85175227986763,
      "learning_rate": 6.201430216410985e-08,
      "logits/chosen": -3.391742706298828,
      "logits/rejected": -3.413292646408081,
      "logps/chosen": -0.5624402761459351,
      "logps/rejected": -0.6359676122665405,
      "loss": 1.5149,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5624402761459351,
      "rewards/margins": 0.07352729141712189,
      "rewards/rejected": -0.6359676122665405,
      "step": 370
    },
    {
      "epoch": 0.8037508372404555,
      "grad_norm": 23.503804219453023,
      "learning_rate": 5.596935348479029e-08,
      "logits/chosen": -3.3563759326934814,
      "logits/rejected": -3.338204860687256,
      "logps/chosen": -0.6166857481002808,
      "logps/rejected": -0.700817883014679,
      "loss": 1.5014,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6166857481002808,
      "rewards/margins": 0.08413223922252655,
      "rewards/rejected": -0.700817883014679,
      "step": 375
    },
    {
      "epoch": 0.8144675150703282,
      "grad_norm": 26.081587652629086,
      "learning_rate": 5.01970709226808e-08,
      "logits/chosen": -3.443342924118042,
      "logits/rejected": -3.4111335277557373,
      "logps/chosen": -0.5469595193862915,
      "logps/rejected": -0.6322815418243408,
      "loss": 1.5083,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5469595193862915,
      "rewards/margins": 0.08532204478979111,
      "rewards/rejected": -0.6322815418243408,
      "step": 380
    },
    {
      "epoch": 0.825184192900201,
      "grad_norm": 24.73716667501714,
      "learning_rate": 4.4705566113246635e-08,
      "logits/chosen": -3.349184513092041,
      "logits/rejected": -3.359894275665283,
      "logps/chosen": -0.5703479647636414,
      "logps/rejected": -0.6489995718002319,
      "loss": 1.4984,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.5703479647636414,
      "rewards/margins": 0.07865166664123535,
      "rewards/rejected": -0.6489995718002319,
      "step": 385
    },
    {
      "epoch": 0.8359008707300737,
      "grad_norm": 24.765231706983972,
      "learning_rate": 3.950255612243042e-08,
      "logits/chosen": -3.3900883197784424,
      "logits/rejected": -3.3493354320526123,
      "logps/chosen": -0.6082748174667358,
      "logps/rejected": -0.6323951482772827,
      "loss": 1.5319,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6082748174667358,
      "rewards/margins": 0.024120373651385307,
      "rewards/rejected": -0.6323951482772827,
      "step": 390
    },
    {
      "epoch": 0.8466175485599464,
      "grad_norm": 23.65912516379117,
      "learning_rate": 3.459535260206384e-08,
      "logits/chosen": -3.3917059898376465,
      "logits/rejected": -3.3759236335754395,
      "logps/chosen": -0.5519801378250122,
      "logps/rejected": -0.5884855389595032,
      "loss": 1.5143,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.5519801378250122,
      "rewards/margins": 0.036505263298749924,
      "rewards/rejected": -0.5884855389595032,
      "step": 395
    },
    {
      "epoch": 0.8573342263898192,
      "grad_norm": 28.498252560499747,
      "learning_rate": 2.999085151499764e-08,
      "logits/chosen": -3.3536014556884766,
      "logits/rejected": -3.322979688644409,
      "logps/chosen": -0.5050632953643799,
      "logps/rejected": -0.6183533072471619,
      "loss": 1.4908,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5050632953643799,
      "rewards/margins": 0.11329002678394318,
      "rewards/rejected": -0.6183533072471619,
      "step": 400
    },
    {
      "epoch": 0.8573342263898192,
      "eval_logits/chosen": -3.4685490131378174,
      "eval_logits/rejected": -3.4632692337036133,
      "eval_logps/chosen": -0.5601765513420105,
      "eval_logps/rejected": -0.6162815093994141,
      "eval_loss": 1.522051215171814,
      "eval_rewards/accuracies": 0.5797872543334961,
      "eval_rewards/chosen": -0.5601765513420105,
      "eval_rewards/margins": 0.05610497295856476,
      "eval_rewards/rejected": -0.6162815093994141,
      "eval_runtime": 433.0685,
      "eval_samples_per_second": 6.913,
      "eval_steps_per_second": 0.434,
      "step": 400
    },
    {
      "epoch": 0.8680509042196919,
      "grad_norm": 23.705058603217022,
      "learning_rate": 2.5695523444387747e-08,
      "logits/chosen": -3.364602565765381,
      "logits/rejected": -3.350884199142456,
      "logps/chosen": -0.5609386563301086,
      "logps/rejected": -0.605690598487854,
      "loss": 1.5111,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.5609386563301086,
      "rewards/margins": 0.044751983135938644,
      "rewards/rejected": -0.605690598487854,
      "step": 405
    },
    {
      "epoch": 0.8787675820495646,
      "grad_norm": 22.37221809399767,
      "learning_rate": 2.1715404500756875e-08,
      "logits/chosen": -3.412458896636963,
      "logits/rejected": -3.3824081420898438,
      "logps/chosen": -0.5534378290176392,
      "logps/rejected": -0.5746613144874573,
      "loss": 1.4932,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.5534378290176392,
      "rewards/margins": 0.021223513409495354,
      "rewards/rejected": -0.5746613144874573,
      "step": 410
    },
    {
      "epoch": 0.8894842598794374,
      "grad_norm": 26.32210656993751,
      "learning_rate": 1.8056087839608546e-08,
      "logits/chosen": -3.332536220550537,
      "logits/rejected": -3.3019680976867676,
      "logps/chosen": -0.5494991540908813,
      "logps/rejected": -0.6182190179824829,
      "loss": 1.4885,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5494991540908813,
      "rewards/margins": 0.06871981918811798,
      "rewards/rejected": -0.6182190179824829,
      "step": 415
    },
    {
      "epoch": 0.9002009377093101,
      "grad_norm": 27.43028943315468,
      "learning_rate": 1.4722715801513935e-08,
      "logits/chosen": -3.3295421600341797,
      "logits/rejected": -3.336806058883667,
      "logps/chosen": -0.5279422998428345,
      "logps/rejected": -0.5645946860313416,
      "loss": 1.5217,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.5279422998428345,
      "rewards/margins": 0.03665238991379738,
      "rewards/rejected": -0.5645946860313416,
      "step": 420
    },
    {
      "epoch": 0.9109176155391828,
      "grad_norm": 25.425067386156872,
      "learning_rate": 1.1719972685717395e-08,
      "logits/chosen": -3.337613582611084,
      "logits/rejected": -3.3247127532958984,
      "logps/chosen": -0.5188074111938477,
      "logps/rejected": -0.6191619634628296,
      "loss": 1.4926,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5188074111938477,
      "rewards/margins": 0.10035452991724014,
      "rewards/rejected": -0.6191619634628296,
      "step": 425
    },
    {
      "epoch": 0.9216342933690556,
      "grad_norm": 35.62756758017028,
      "learning_rate": 9.05207816741485e-09,
      "logits/chosen": -3.3888893127441406,
      "logits/rejected": -3.4012131690979004,
      "logps/chosen": -0.6197360754013062,
      "logps/rejected": -0.6971450448036194,
      "loss": 1.5011,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6197360754013062,
      "rewards/margins": 0.07740898430347443,
      "rewards/rejected": -0.6971450448036194,
      "step": 430
    },
    {
      "epoch": 0.9323509711989283,
      "grad_norm": 27.22699922034511,
      "learning_rate": 6.722781367956481e-09,
      "logits/chosen": -3.455385208129883,
      "logits/rejected": -3.4597179889678955,
      "logps/chosen": -0.5144860744476318,
      "logps/rejected": -0.6305041909217834,
      "loss": 1.4974,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.5144860744476318,
      "rewards/margins": 0.1160181388258934,
      "rewards/rejected": -0.6305041909217834,
      "step": 435
    },
    {
      "epoch": 0.943067649028801,
      "grad_norm": 23.43687138581986,
      "learning_rate": 4.735355586305861e-09,
      "logits/chosen": -3.2910914421081543,
      "logits/rejected": -3.288029193878174,
      "logps/chosen": -0.5328935980796814,
      "logps/rejected": -0.6186143755912781,
      "loss": 1.4731,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5328935980796814,
      "rewards/margins": 0.08572079241275787,
      "rewards/rejected": -0.6186143755912781,
      "step": 440
    },
    {
      "epoch": 0.9537843268586738,
      "grad_norm": 27.25503683317219,
      "learning_rate": 3.0925936991598454e-09,
      "logits/chosen": -3.320718765258789,
      "logits/rejected": -3.319967746734619,
      "logps/chosen": -0.5472747683525085,
      "logps/rejected": -0.6223639845848083,
      "loss": 1.493,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.5472747683525085,
      "rewards/margins": 0.07508920133113861,
      "rewards/rejected": -0.6223639845848083,
      "step": 445
    },
    {
      "epoch": 0.9645010046885466,
      "grad_norm": 23.669876406776968,
      "learning_rate": 1.7968042361930225e-09,
      "logits/chosen": -3.354473114013672,
      "logits/rejected": -3.3409972190856934,
      "logps/chosen": -0.5448306202888489,
      "logps/rejected": -0.6130356788635254,
      "loss": 1.4876,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.5448306202888489,
      "rewards/margins": 0.06820502877235413,
      "rewards/rejected": -0.6130356788635254,
      "step": 450
    },
    {
      "epoch": 0.9752176825184193,
      "grad_norm": 27.737663302659318,
      "learning_rate": 8.498081359423759e-10,
      "logits/chosen": -3.331641435623169,
      "logits/rejected": -3.3293700218200684,
      "logps/chosen": -0.5374922156333923,
      "logps/rejected": -0.6145657300949097,
      "loss": 1.4953,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5374922156333923,
      "rewards/margins": 0.07707355916500092,
      "rewards/rejected": -0.6145657300949097,
      "step": 455
    },
    {
      "epoch": 0.9859343603482921,
      "grad_norm": 29.309243600698153,
      "learning_rate": 2.5293618689056774e-10,
      "logits/chosen": -3.3501858711242676,
      "logits/rejected": -3.3541958332061768,
      "logps/chosen": -0.5475316643714905,
      "logps/rejected": -0.6366786360740662,
      "loss": 1.5039,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5475316643714905,
      "rewards/margins": 0.08914700895547867,
      "rewards/rejected": -0.6366786360740662,
      "step": 460
    },
    {
      "epoch": 0.9966510381781648,
      "grad_norm": 21.403219882354602,
      "learning_rate": 7.02715734424153e-12,
      "logits/chosen": -3.3826191425323486,
      "logits/rejected": -3.36454701423645,
      "logps/chosen": -0.5860726833343506,
      "logps/rejected": -0.6269819736480713,
      "loss": 1.5076,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.5860726833343506,
      "rewards/margins": 0.040909312665462494,
      "rewards/rejected": -0.6269819736480713,
      "step": 465
    },
    {
      "epoch": 0.9987943737441393,
      "step": 466,
      "total_flos": 0.0,
      "train_loss": 1.5188358733070766,
      "train_runtime": 19352.2078,
      "train_samples_per_second": 3.086,
      "train_steps_per_second": 0.024
    }
  ],
  "logging_steps": 5,
  "max_steps": 466,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}