{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9981298423724285,
  "eval_steps": 100,
  "global_step": 467,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0021373230029388193,
      "grad_norm": 5.562169562786842,
      "learning_rate": 1.4893617021276595e-08,
      "logits/chosen": -0.9639219045639038,
      "logits/rejected": -0.9681637287139893,
      "logps/chosen": -254.6912078857422,
      "logps/rejected": -263.5589904785156,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.004274646005877639,
      "grad_norm": 5.450588274868951,
      "learning_rate": 2.978723404255319e-08,
      "logits/chosen": -0.9656609296798706,
      "logits/rejected": -0.9244194030761719,
      "logps/chosen": -326.766845703125,
      "logps/rejected": -278.7445068359375,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.006411969008816457,
      "grad_norm": 5.947914793856331,
      "learning_rate": 4.468085106382978e-08,
      "logits/chosen": -0.8438385725021362,
      "logits/rejected": -0.8096433877944946,
      "logps/chosen": -235.20404052734375,
      "logps/rejected": -236.10145568847656,
      "loss": 0.6932,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0013289405032992363,
      "rewards/margins": -0.00010445114457979798,
      "rewards/rejected": 0.0014333915896713734,
      "step": 3
    },
    {
      "epoch": 0.008549292011755277,
      "grad_norm": 5.296514371602818,
      "learning_rate": 5.957446808510638e-08,
      "logits/chosen": -0.9574068784713745,
      "logits/rejected": -0.9784606099128723,
      "logps/chosen": -205.1729736328125,
      "logps/rejected": -211.75196838378906,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0005042314296588302,
      "rewards/margins": -0.00139321805909276,
      "rewards/rejected": 0.0008889864548109472,
      "step": 4
    },
    {
      "epoch": 0.010686615014694095,
      "grad_norm": 5.901392519236585,
      "learning_rate": 7.446808510638298e-08,
      "logits/chosen": -1.0202012062072754,
      "logits/rejected": -1.05290687084198,
      "logps/chosen": -223.4305419921875,
      "logps/rejected": -223.0294647216797,
      "loss": 0.6933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0012667321134358644,
      "rewards/margins": 0.00010157108772546053,
      "rewards/rejected": 0.0011651611421257257,
      "step": 5
    },
    {
      "epoch": 0.012823938017632914,
      "grad_norm": 6.106968331942851,
      "learning_rate": 8.936170212765956e-08,
      "logits/chosen": -1.01454496383667,
      "logits/rejected": -0.9522863626480103,
      "logps/chosen": -310.51141357421875,
      "logps/rejected": -288.7611083984375,
      "loss": 0.6928,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0014815283939242363,
      "rewards/margins": -0.0021853544749319553,
      "rewards/rejected": 0.0007038259063847363,
      "step": 6
    },
    {
      "epoch": 0.014961261020571734,
      "grad_norm": 4.989592008812262,
      "learning_rate": 1.0425531914893615e-07,
      "logits/chosen": -1.084510087966919,
      "logits/rejected": -1.029686689376831,
      "logps/chosen": -236.44857788085938,
      "logps/rejected": -227.99609375,
      "loss": 0.6936,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0002356480690650642,
      "rewards/margins": 0.001065850374288857,
      "rewards/rejected": -0.0013014983851462603,
      "step": 7
    },
    {
      "epoch": 0.017098584023510555,
      "grad_norm": 5.864652813608174,
      "learning_rate": 1.1914893617021276e-07,
      "logits/chosen": -1.0324242115020752,
      "logits/rejected": -0.9804142713546753,
      "logps/chosen": -248.21893310546875,
      "logps/rejected": -262.17919921875,
      "loss": 0.6934,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0010573435574769974,
      "rewards/margins": 7.791988173266873e-05,
      "rewards/rejected": -0.001135263592004776,
      "step": 8
    },
    {
      "epoch": 0.01923590702644937,
      "grad_norm": 5.012836085935026,
      "learning_rate": 1.3404255319148934e-07,
      "logits/chosen": -0.9063066244125366,
      "logits/rejected": -0.888552188873291,
      "logps/chosen": -281.49481201171875,
      "logps/rejected": -285.80462646484375,
      "loss": 0.6932,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0012888526543974876,
      "rewards/margins": -0.00021493433450814337,
      "rewards/rejected": 0.0015037870034575462,
      "step": 9
    },
    {
      "epoch": 0.02137323002938819,
      "grad_norm": 5.441686883546068,
      "learning_rate": 1.4893617021276595e-07,
      "logits/chosen": -1.1913858652114868,
      "logits/rejected": -1.1670382022857666,
      "logps/chosen": -264.4725646972656,
      "logps/rejected": -285.6479797363281,
      "loss": 0.6932,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0027979514561593533,
      "rewards/margins": 0.0013645172584801912,
      "rewards/rejected": 0.0014334343140944839,
      "step": 10
    },
    {
      "epoch": 0.02351055303232701,
      "grad_norm": 5.606555817511537,
      "learning_rate": 1.6382978723404256e-07,
      "logits/chosen": -1.0321263074874878,
      "logits/rejected": -1.0099542140960693,
      "logps/chosen": -266.173828125,
      "logps/rejected": -278.39117431640625,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -5.334848538041115e-05,
      "rewards/margins": 0.0002821112284436822,
      "rewards/rejected": -0.00033545983023941517,
      "step": 11
    },
    {
      "epoch": 0.02564787603526583,
      "grad_norm": 4.68531837452694,
      "learning_rate": 1.7872340425531912e-07,
      "logits/chosen": -0.9241148233413696,
      "logits/rejected": -0.9032956957817078,
      "logps/chosen": -232.3054962158203,
      "logps/rejected": -213.7734375,
      "loss": 0.6926,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0020639896392822266,
      "rewards/margins": 0.00174471375066787,
      "rewards/rejected": 0.000319275917718187,
      "step": 12
    },
    {
      "epoch": 0.027785199038204648,
      "grad_norm": 6.413087644872293,
      "learning_rate": 1.9361702127659575e-07,
      "logits/chosen": -0.8520964980125427,
      "logits/rejected": -0.7619133591651917,
      "logps/chosen": -257.8199462890625,
      "logps/rejected": -286.0062255859375,
      "loss": 0.6932,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0032129527535289526,
      "rewards/margins": 0.004196567460894585,
      "rewards/rejected": -0.007409519515931606,
      "step": 13
    },
    {
      "epoch": 0.029922522041143467,
      "grad_norm": 6.109488078686547,
      "learning_rate": 2.085106382978723e-07,
      "logits/chosen": -1.0061469078063965,
      "logits/rejected": -1.0426419973373413,
      "logps/chosen": -300.9575500488281,
      "logps/rejected": -284.1053466796875,
      "loss": 0.6933,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0033643674105405807,
      "rewards/margins": -0.002198734087869525,
      "rewards/rejected": -0.001165633206255734,
      "step": 14
    },
    {
      "epoch": 0.03205984504408229,
      "grad_norm": 5.1906516925117225,
      "learning_rate": 2.2340425531914894e-07,
      "logits/chosen": -1.1642401218414307,
      "logits/rejected": -1.1202101707458496,
      "logps/chosen": -264.85552978515625,
      "logps/rejected": -270.9376525878906,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0047036223113536835,
      "rewards/margins": 0.0003113269922323525,
      "rewards/rejected": -0.005014949012547731,
      "step": 15
    },
    {
      "epoch": 0.03419716804702111,
      "grad_norm": 4.9412589689200095,
      "learning_rate": 2.3829787234042553e-07,
      "logits/chosen": -1.0979671478271484,
      "logits/rejected": -1.067629098892212,
      "logps/chosen": -271.14385986328125,
      "logps/rejected": -280.5599670410156,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -9.246054105460644e-06,
      "rewards/margins": 0.0020150854252278805,
      "rewards/rejected": -0.0020243311300873756,
      "step": 16
    },
    {
      "epoch": 0.03633449104995993,
      "grad_norm": 5.374113235339209,
      "learning_rate": 2.5319148936170213e-07,
      "logits/chosen": -1.1378012895584106,
      "logits/rejected": -1.087912678718567,
      "logps/chosen": -237.7337188720703,
      "logps/rejected": -237.0882110595703,
      "loss": 0.6931,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003059606533497572,
      "rewards/margins": -4.628184251487255e-05,
      "rewards/rejected": -0.003013324923813343,
      "step": 17
    },
    {
      "epoch": 0.03847181405289874,
      "grad_norm": 6.092267360675494,
      "learning_rate": 2.680851063829787e-07,
      "logits/chosen": -1.0425890684127808,
      "logits/rejected": -1.0599486827850342,
      "logps/chosen": -242.11541748046875,
      "logps/rejected": -256.46417236328125,
      "loss": 0.6928,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004783968441188335,
      "rewards/margins": -9.258274803869426e-05,
      "rewards/rejected": -0.004691386129707098,
      "step": 18
    },
    {
      "epoch": 0.04060913705583756,
      "grad_norm": 5.1497648303931784,
      "learning_rate": 2.829787234042553e-07,
      "logits/chosen": -1.1328908205032349,
      "logits/rejected": -1.0197917222976685,
      "logps/chosen": -253.92127990722656,
      "logps/rejected": -278.0308837890625,
      "loss": 0.6923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0021464251913130283,
      "rewards/margins": 0.00046774852671660483,
      "rewards/rejected": -0.0026141738053411245,
      "step": 19
    },
    {
      "epoch": 0.04274646005877638,
      "grad_norm": 5.9119652568805225,
      "learning_rate": 2.978723404255319e-07,
      "logits/chosen": -1.1393866539001465,
      "logits/rejected": -1.0607531070709229,
      "logps/chosen": -303.4728088378906,
      "logps/rejected": -295.6599426269531,
      "loss": 0.6916,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.004760389216244221,
      "rewards/margins": 0.003504714695736766,
      "rewards/rejected": -0.00826510414481163,
      "step": 20
    },
    {
      "epoch": 0.0448837830617152,
      "grad_norm": 5.752883808298607,
      "learning_rate": 3.1276595744680846e-07,
      "logits/chosen": -1.1522903442382812,
      "logits/rejected": -1.1488419771194458,
      "logps/chosen": -192.28042602539062,
      "logps/rejected": -200.01608276367188,
      "loss": 0.6911,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004406332969665527,
      "rewards/margins": -0.0005414390470832586,
      "rewards/rejected": -0.0038648939225822687,
      "step": 21
    },
    {
      "epoch": 0.04702110606465402,
      "grad_norm": 4.856538956353405,
      "learning_rate": 3.276595744680851e-07,
      "logits/chosen": -1.0699973106384277,
      "logits/rejected": -1.0846004486083984,
      "logps/chosen": -205.7268829345703,
      "logps/rejected": -201.65975952148438,
      "loss": 0.6915,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005930948071181774,
      "rewards/margins": 0.0032684658654034138,
      "rewards/rejected": -0.009199414402246475,
      "step": 22
    },
    {
      "epoch": 0.04915842906759284,
      "grad_norm": 4.906648390477566,
      "learning_rate": 3.425531914893617e-07,
      "logits/chosen": -0.9317066669464111,
      "logits/rejected": -0.8009700179100037,
      "logps/chosen": -246.80438232421875,
      "logps/rejected": -272.9093322753906,
      "loss": 0.6911,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.010996412485837936,
      "rewards/margins": 0.0014129301998764277,
      "rewards/rejected": -0.012409343384206295,
      "step": 23
    },
    {
      "epoch": 0.05129575207053166,
      "grad_norm": 5.875935612586806,
      "learning_rate": 3.5744680851063824e-07,
      "logits/chosen": -1.069819688796997,
      "logits/rejected": -0.9709153175354004,
      "logps/chosen": -195.9522247314453,
      "logps/rejected": -229.61566162109375,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004574608989059925,
      "rewards/margins": -0.0003843879676423967,
      "rewards/rejected": -0.00419022049754858,
      "step": 24
    },
    {
      "epoch": 0.053433075073470476,
      "grad_norm": 4.985096243829474,
      "learning_rate": 3.7234042553191484e-07,
      "logits/chosen": -0.8998661637306213,
      "logits/rejected": -0.8598207831382751,
      "logps/chosen": -282.3105773925781,
      "logps/rejected": -272.4259338378906,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010662160813808441,
      "rewards/margins": 0.005043415818363428,
      "rewards/rejected": -0.015705574303865433,
      "step": 25
    },
    {
      "epoch": 0.055570398076409296,
      "grad_norm": 5.218346832933627,
      "learning_rate": 3.872340425531915e-07,
      "logits/chosen": -0.9051691889762878,
      "logits/rejected": -0.8448677659034729,
      "logps/chosen": -241.62689208984375,
      "logps/rejected": -257.25244140625,
      "loss": 0.6901,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02060263603925705,
      "rewards/margins": -0.0019798323046416044,
      "rewards/rejected": -0.018622804433107376,
      "step": 26
    },
    {
      "epoch": 0.057707721079348115,
      "grad_norm": 5.189530040363191,
      "learning_rate": 4.021276595744681e-07,
      "logits/chosen": -1.0560821294784546,
      "logits/rejected": -1.0800153017044067,
      "logps/chosen": -252.28952026367188,
      "logps/rejected": -245.88502502441406,
      "loss": 0.6898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.015861667692661285,
      "rewards/margins": 0.007801623083651066,
      "rewards/rejected": -0.023663289844989777,
      "step": 27
    },
    {
      "epoch": 0.059845044082286934,
      "grad_norm": 5.8226698693397,
      "learning_rate": 4.170212765957446e-07,
      "logits/chosen": -1.0804771184921265,
      "logits/rejected": -1.099622130393982,
      "logps/chosen": -227.35537719726562,
      "logps/rejected": -225.36538696289062,
      "loss": 0.6887,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.017352018505334854,
      "rewards/margins": 0.01319974847137928,
      "rewards/rejected": -0.030551765114068985,
      "step": 28
    },
    {
      "epoch": 0.061982367085225754,
      "grad_norm": 5.460690205724815,
      "learning_rate": 4.319148936170213e-07,
      "logits/chosen": -1.0153366327285767,
      "logits/rejected": -1.0103431940078735,
      "logps/chosen": -263.8896789550781,
      "logps/rejected": -267.3290100097656,
      "loss": 0.6878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02432805672287941,
      "rewards/margins": 0.008317284286022186,
      "rewards/rejected": -0.032645341008901596,
      "step": 29
    },
    {
      "epoch": 0.06411969008816458,
      "grad_norm": 5.737017895955914,
      "learning_rate": 4.468085106382979e-07,
      "logits/chosen": -1.0464987754821777,
      "logits/rejected": -1.1331242322921753,
      "logps/chosen": -206.78468322753906,
      "logps/rejected": -192.26502990722656,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010598372668027878,
      "rewards/margins": 0.002255907515063882,
      "rewards/rejected": -0.012854279950261116,
      "step": 30
    },
    {
      "epoch": 0.06625701309110339,
      "grad_norm": 5.795133035841147,
      "learning_rate": 4.617021276595744e-07,
      "logits/chosen": -0.9297857880592346,
      "logits/rejected": -0.8592264652252197,
      "logps/chosen": -227.1981964111328,
      "logps/rejected": -247.9413299560547,
      "loss": 0.6863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03003655932843685,
      "rewards/margins": 0.009244289249181747,
      "rewards/rejected": -0.03928084671497345,
      "step": 31
    },
    {
      "epoch": 0.06839433609404222,
      "grad_norm": 6.814768956718574,
      "learning_rate": 4.7659574468085105e-07,
      "logits/chosen": -1.05179762840271,
      "logits/rejected": -1.0814404487609863,
      "logps/chosen": -290.69525146484375,
      "logps/rejected": -274.92047119140625,
      "loss": 0.6818,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05112060159444809,
      "rewards/margins": 0.015534894540905952,
      "rewards/rejected": -0.06665549427270889,
      "step": 32
    },
    {
      "epoch": 0.07053165909698103,
      "grad_norm": 6.074153638196805,
      "learning_rate": 4.914893617021277e-07,
      "logits/chosen": -0.8849226236343384,
      "logits/rejected": -0.9219367504119873,
      "logps/chosen": -317.4134826660156,
      "logps/rejected": -325.8724365234375,
      "loss": 0.6832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04802538454532623,
      "rewards/margins": 0.024860400706529617,
      "rewards/rejected": -0.07288578897714615,
      "step": 33
    },
    {
      "epoch": 0.07266898209991986,
      "grad_norm": 6.0900485696331215,
      "learning_rate": 5.063829787234043e-07,
      "logits/chosen": -1.0383641719818115,
      "logits/rejected": -1.0028424263000488,
      "logps/chosen": -277.57965087890625,
      "logps/rejected": -268.2386474609375,
      "loss": 0.6848,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.060051530599594116,
      "rewards/margins": 0.0026317029260098934,
      "rewards/rejected": -0.06268322467803955,
      "step": 34
    },
    {
      "epoch": 0.07480630510285867,
      "grad_norm": 6.827504173132289,
      "learning_rate": 5.212765957446808e-07,
      "logits/chosen": -1.0755884647369385,
      "logits/rejected": -1.0121068954467773,
      "logps/chosen": -311.99755859375,
      "logps/rejected": -350.29052734375,
      "loss": 0.682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.061084531247615814,
      "rewards/margins": 0.04312053322792053,
      "rewards/rejected": -0.10420507192611694,
      "step": 35
    },
    {
      "epoch": 0.07694362810579748,
      "grad_norm": 6.012937319519413,
      "learning_rate": 5.361702127659574e-07,
      "logits/chosen": -1.1560208797454834,
      "logits/rejected": -1.0447015762329102,
      "logps/chosen": -216.53421020507812,
      "logps/rejected": -234.53128051757812,
      "loss": 0.6788,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04387297108769417,
      "rewards/margins": 0.016591297462582588,
      "rewards/rejected": -0.06046426668763161,
      "step": 36
    },
    {
      "epoch": 0.07908095110873631,
      "grad_norm": 5.599688064843719,
      "learning_rate": 5.51063829787234e-07,
      "logits/chosen": -1.2253515720367432,
      "logits/rejected": -1.2332267761230469,
      "logps/chosen": -237.71002197265625,
      "logps/rejected": -236.66990661621094,
      "loss": 0.6799,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08036091923713684,
      "rewards/margins": 0.03013552539050579,
      "rewards/rejected": -0.11049643903970718,
      "step": 37
    },
    {
      "epoch": 0.08121827411167512,
      "grad_norm": 6.7746928876503105,
      "learning_rate": 5.659574468085106e-07,
      "logits/chosen": -1.0830399990081787,
      "logits/rejected": -1.0622460842132568,
      "logps/chosen": -284.8079833984375,
      "logps/rejected": -272.3097839355469,
      "loss": 0.6755,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07364249229431152,
      "rewards/margins": 0.03328356519341469,
      "rewards/rejected": -0.10692604631185532,
      "step": 38
    },
    {
      "epoch": 0.08335559711461395,
      "grad_norm": 6.77614986840528,
      "learning_rate": 5.808510638297872e-07,
      "logits/chosen": -1.0905030965805054,
      "logits/rejected": -1.0605907440185547,
      "logps/chosen": -210.57684326171875,
      "logps/rejected": -239.24871826171875,
      "loss": 0.6734,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1003848984837532,
      "rewards/margins": 0.024850603193044662,
      "rewards/rejected": -0.12523549795150757,
      "step": 39
    },
    {
      "epoch": 0.08549292011755276,
      "grad_norm": 6.424490447178767,
      "learning_rate": 5.957446808510638e-07,
      "logits/chosen": -1.1801645755767822,
      "logits/rejected": -1.1392816305160522,
      "logps/chosen": -263.45037841796875,
      "logps/rejected": -273.5709533691406,
      "loss": 0.6736,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11433250457048416,
      "rewards/margins": 0.028288790956139565,
      "rewards/rejected": -0.14262130856513977,
      "step": 40
    },
    {
      "epoch": 0.08763024312049159,
      "grad_norm": 6.4654988266248425,
      "learning_rate": 6.106382978723404e-07,
      "logits/chosen": -1.0815637111663818,
      "logits/rejected": -1.013136625289917,
      "logps/chosen": -206.23907470703125,
      "logps/rejected": -220.0244140625,
      "loss": 0.6722,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11655673384666443,
      "rewards/margins": 0.04582422599196434,
      "rewards/rejected": -0.16238096356391907,
      "step": 41
    },
    {
      "epoch": 0.0897675661234304,
      "grad_norm": 6.539055183308233,
      "learning_rate": 6.255319148936169e-07,
      "logits/chosen": -1.077747106552124,
      "logits/rejected": -1.0630099773406982,
      "logps/chosen": -268.242431640625,
      "logps/rejected": -304.5572204589844,
      "loss": 0.6716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1858682483434677,
      "rewards/margins": 0.05748748779296875,
      "rewards/rejected": -0.24335573613643646,
      "step": 42
    },
    {
      "epoch": 0.09190488912636922,
      "grad_norm": 6.095364046949072,
      "learning_rate": 6.404255319148935e-07,
      "logits/chosen": -1.0136995315551758,
      "logits/rejected": -0.9615681171417236,
      "logps/chosen": -228.80368041992188,
      "logps/rejected": -235.71481323242188,
      "loss": 0.6689,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1476023644208908,
      "rewards/margins": 0.0479072704911232,
      "rewards/rejected": -0.1955096423625946,
      "step": 43
    },
    {
      "epoch": 0.09404221212930804,
      "grad_norm": 6.7344523666741765,
      "learning_rate": 6.553191489361702e-07,
      "logits/chosen": -0.8863506317138672,
      "logits/rejected": -0.9662103652954102,
      "logps/chosen": -259.41595458984375,
      "logps/rejected": -272.3363037109375,
      "loss": 0.6688,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.19808562099933624,
      "rewards/margins": 0.013540804386138916,
      "rewards/rejected": -0.21162641048431396,
      "step": 44
    },
    {
      "epoch": 0.09617953513224686,
      "grad_norm": 6.29136903272874,
      "learning_rate": 6.702127659574469e-07,
      "logits/chosen": -1.0862454175949097,
      "logits/rejected": -1.03424870967865,
      "logps/chosen": -343.55059814453125,
      "logps/rejected": -352.5019836425781,
      "loss": 0.6619,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25226354598999023,
      "rewards/margins": 0.05790935456752777,
      "rewards/rejected": -0.3101729154586792,
      "step": 45
    },
    {
      "epoch": 0.09831685813518568,
      "grad_norm": 5.905042821290849,
      "learning_rate": 6.851063829787234e-07,
      "logits/chosen": -1.086181402206421,
      "logits/rejected": -1.0607441663742065,
      "logps/chosen": -281.6853942871094,
      "logps/rejected": -306.71136474609375,
      "loss": 0.6646,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.30046984553337097,
      "rewards/margins": 0.11004211008548737,
      "rewards/rejected": -0.41051194071769714,
      "step": 46
    },
    {
      "epoch": 0.1004541811381245,
      "grad_norm": 6.664836667296562,
      "learning_rate": 7e-07,
      "logits/chosen": -1.1094589233398438,
      "logits/rejected": -1.0737080574035645,
      "logps/chosen": -276.6365966796875,
      "logps/rejected": -290.7496032714844,
      "loss": 0.6509,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2919345498085022,
      "rewards/margins": 0.12451538443565369,
      "rewards/rejected": -0.41644996404647827,
      "step": 47
    },
    {
      "epoch": 0.10259150414106331,
      "grad_norm": 6.452030342745469,
      "learning_rate": 6.999902087714444e-07,
      "logits/chosen": -1.0068883895874023,
      "logits/rejected": -0.955917239189148,
      "logps/chosen": -222.2923583984375,
      "logps/rejected": -239.01463317871094,
      "loss": 0.6577,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1943490207195282,
      "rewards/margins": 0.08093887567520142,
      "rewards/rejected": -0.275287926197052,
      "step": 48
    },
    {
      "epoch": 0.10472882714400214,
      "grad_norm": 7.754778118919807,
      "learning_rate": 6.999608356335957e-07,
      "logits/chosen": -1.0677528381347656,
      "logits/rejected": -1.0721509456634521,
      "logps/chosen": -227.729248046875,
      "logps/rejected": -239.76138305664062,
      "loss": 0.645,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2937619090080261,
      "rewards/margins": 0.048121094703674316,
      "rewards/rejected": -0.3418830335140228,
      "step": 49
    },
    {
      "epoch": 0.10686615014694095,
      "grad_norm": 6.437697145095338,
      "learning_rate": 6.999118822298773e-07,
      "logits/chosen": -1.1454989910125732,
      "logits/rejected": -1.0754855871200562,
      "logps/chosen": -227.07974243164062,
      "logps/rejected": -228.60333251953125,
      "loss": 0.6547,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2852881848812103,
      "rewards/margins": 0.0602235309779644,
      "rewards/rejected": -0.3455117344856262,
      "step": 50
    },
    {
      "epoch": 0.10900347314987978,
      "grad_norm": 7.527535438459104,
      "learning_rate": 6.998433512992261e-07,
      "logits/chosen": -1.1637341976165771,
      "logits/rejected": -1.0740972757339478,
      "logps/chosen": -292.79742431640625,
      "logps/rejected": -302.4303283691406,
      "loss": 0.6493,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5110954642295837,
      "rewards/margins": 0.09770005941390991,
      "rewards/rejected": -0.6087955236434937,
      "step": 51
    },
    {
      "epoch": 0.11114079615281859,
      "grad_norm": 7.575162846371126,
      "learning_rate": 6.997552466759395e-07,
      "logits/chosen": -1.0683425664901733,
      "logits/rejected": -1.1101981401443481,
      "logps/chosen": -297.38702392578125,
      "logps/rejected": -309.9881286621094,
      "loss": 0.6288,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4417276978492737,
      "rewards/margins": 0.09606233239173889,
      "rewards/rejected": -0.537790060043335,
      "step": 52
    },
    {
      "epoch": 0.11327811915575742,
      "grad_norm": 7.584356018239797,
      "learning_rate": 6.996475732894601e-07,
      "logits/chosen": -1.0953909158706665,
      "logits/rejected": -1.0685151815414429,
      "logps/chosen": -336.0571594238281,
      "logps/rejected": -369.7088623046875,
      "loss": 0.6369,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6481105089187622,
      "rewards/margins": 0.13110791146755219,
      "rewards/rejected": -0.779218316078186,
      "step": 53
    },
    {
      "epoch": 0.11541544215869623,
      "grad_norm": 6.778514980526427,
      "learning_rate": 6.995203371641008e-07,
      "logits/chosen": -1.1196974515914917,
      "logits/rejected": -1.0348316431045532,
      "logps/chosen": -287.28729248046875,
      "logps/rejected": -302.94317626953125,
      "loss": 0.6444,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5020595788955688,
      "rewards/margins": 0.09727220237255096,
      "rewards/rejected": -0.5993317365646362,
      "step": 54
    },
    {
      "epoch": 0.11755276516163506,
      "grad_norm": 8.211287073219191,
      "learning_rate": 6.993735454187072e-07,
      "logits/chosen": -1.0214238166809082,
      "logits/rejected": -0.9623696804046631,
      "logps/chosen": -265.59417724609375,
      "logps/rejected": -298.86444091796875,
      "loss": 0.6016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5154373645782471,
      "rewards/margins": 0.24744802713394165,
      "rewards/rejected": -0.7628854513168335,
      "step": 55
    },
    {
      "epoch": 0.11969008816457387,
      "grad_norm": 7.2415969346538756,
      "learning_rate": 6.992072062662596e-07,
      "logits/chosen": -0.9752060174942017,
      "logits/rejected": -0.9751684069633484,
      "logps/chosen": -349.1664123535156,
      "logps/rejected": -362.6107177734375,
      "loss": 0.6343,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7859246134757996,
      "rewards/margins": 0.13539882004261017,
      "rewards/rejected": -0.9213234186172485,
      "step": 56
    },
    {
      "epoch": 0.1218274111675127,
      "grad_norm": 6.635074198180161,
      "learning_rate": 6.99021329013413e-07,
      "logits/chosen": -1.1370452642440796,
      "logits/rejected": -1.0629245042800903,
      "logps/chosen": -288.10430908203125,
      "logps/rejected": -307.46484375,
      "loss": 0.61,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6563497185707092,
      "rewards/margins": 0.21564731001853943,
      "rewards/rejected": -0.8719969987869263,
      "step": 57
    },
    {
      "epoch": 0.12396473417045151,
      "grad_norm": 7.22705417609884,
      "learning_rate": 6.988159240599771e-07,
      "logits/chosen": -1.1842763423919678,
      "logits/rejected": -1.1428574323654175,
      "logps/chosen": -288.5982666015625,
      "logps/rejected": -291.8437194824219,
      "loss": 0.6186,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7895088195800781,
      "rewards/margins": 0.07229967415332794,
      "rewards/rejected": -0.8618084788322449,
      "step": 58
    },
    {
      "epoch": 0.12610205717339032,
      "grad_norm": 7.191134813555289,
      "learning_rate": 6.985910028983336e-07,
      "logits/chosen": -1.0885567665100098,
      "logits/rejected": -1.0549125671386719,
      "logps/chosen": -484.6271667480469,
      "logps/rejected": -517.6636352539062,
      "loss": 0.6022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1380163431167603,
      "rewards/margins": 0.33240166306495667,
      "rewards/rejected": -1.4704179763793945,
      "step": 59
    },
    {
      "epoch": 0.12823938017632916,
      "grad_norm": 6.856697802256405,
      "learning_rate": 6.983465781127942e-07,
      "logits/chosen": -1.0434743165969849,
      "logits/rejected": -1.0580846071243286,
      "logps/chosen": -315.3133239746094,
      "logps/rejected": -314.5843811035156,
      "loss": 0.6142,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7601010799407959,
      "rewards/margins": 0.006043531000614166,
      "rewards/rejected": -0.7661447525024414,
      "step": 60
    },
    {
      "epoch": 0.13037670317926797,
      "grad_norm": 6.8390931208538985,
      "learning_rate": 6.980826633788957e-07,
      "logits/chosen": -1.0378106832504272,
      "logits/rejected": -0.9883963465690613,
      "logps/chosen": -406.2007751464844,
      "logps/rejected": -471.77471923828125,
      "loss": 0.611,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.0759634971618652,
      "rewards/margins": 0.3877870440483093,
      "rewards/rejected": -1.4637506008148193,
      "step": 61
    },
    {
      "epoch": 0.13251402618220678,
      "grad_norm": 8.256162255166101,
      "learning_rate": 6.977992734626349e-07,
      "logits/chosen": -1.2170382738113403,
      "logits/rejected": -1.1635799407958984,
      "logps/chosen": -409.862548828125,
      "logps/rejected": -459.107421875,
      "loss": 0.5815,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3327953815460205,
      "rewards/margins": 0.5586321353912354,
      "rewards/rejected": -1.8914276361465454,
      "step": 62
    },
    {
      "epoch": 0.1346513491851456,
      "grad_norm": 6.7905205812341185,
      "learning_rate": 6.97496424219643e-07,
      "logits/chosen": -0.9324991106987,
      "logits/rejected": -0.9242455959320068,
      "logps/chosen": -305.8074035644531,
      "logps/rejected": -359.38275146484375,
      "loss": 0.5897,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7552638649940491,
      "rewards/margins": 0.3947301506996155,
      "rewards/rejected": -1.1499940156936646,
      "step": 63
    },
    {
      "epoch": 0.13678867218808444,
      "grad_norm": 7.929159551201501,
      "learning_rate": 6.971741325942982e-07,
      "logits/chosen": -1.0476861000061035,
      "logits/rejected": -1.0574254989624023,
      "logps/chosen": -397.0540466308594,
      "logps/rejected": -445.9757385253906,
      "loss": 0.5989,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1025577783584595,
      "rewards/margins": 0.5505799055099487,
      "rewards/rejected": -1.6531375646591187,
      "step": 64
    },
    {
      "epoch": 0.13892599519102325,
      "grad_norm": 6.60300962363944,
      "learning_rate": 6.968324166187772e-07,
      "logits/chosen": -1.0173978805541992,
      "logits/rejected": -1.0060231685638428,
      "logps/chosen": -391.6329345703125,
      "logps/rejected": -482.0704040527344,
      "loss": 0.5949,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2589941024780273,
      "rewards/margins": 0.7740709781646729,
      "rewards/rejected": -2.0330650806427,
      "step": 65
    },
    {
      "epoch": 0.14106331819396206,
      "grad_norm": 7.722534638937169,
      "learning_rate": 6.96471295412047e-07,
      "logits/chosen": -1.210124135017395,
      "logits/rejected": -1.1669820547103882,
      "logps/chosen": -436.8033447265625,
      "logps/rejected": -447.0545349121094,
      "loss": 0.6197,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.7195076942443848,
      "rewards/margins": 0.12996703386306763,
      "rewards/rejected": -1.8494747877120972,
      "step": 66
    },
    {
      "epoch": 0.14320064119690087,
      "grad_norm": 8.147380246018754,
      "learning_rate": 6.960907891787949e-07,
      "logits/chosen": -1.165956974029541,
      "logits/rejected": -1.1428234577178955,
      "logps/chosen": -519.3298950195312,
      "logps/rejected": -559.141357421875,
      "loss": 0.547,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.083847999572754,
      "rewards/margins": 0.4430221617221832,
      "rewards/rejected": -2.5268702507019043,
      "step": 67
    },
    {
      "epoch": 0.14533796419983971,
      "grad_norm": 7.8944938583131945,
      "learning_rate": 6.956909192082982e-07,
      "logits/chosen": -0.9343075752258301,
      "logits/rejected": -0.8972913026809692,
      "logps/chosen": -292.84552001953125,
      "logps/rejected": -341.25872802734375,
      "loss": 0.5741,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.985328197479248,
      "rewards/margins": 0.39189204573631287,
      "rewards/rejected": -1.3772202730178833,
      "step": 68
    },
    {
      "epoch": 0.14747528720277853,
      "grad_norm": 7.019821529750172,
      "learning_rate": 6.952717078732324e-07,
      "logits/chosen": -1.0702323913574219,
      "logits/rejected": -0.9579718112945557,
      "logps/chosen": -378.2999572753906,
      "logps/rejected": -452.3642578125,
      "loss": 0.6007,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3012688159942627,
      "rewards/margins": 0.5469968318939209,
      "rewards/rejected": -1.8482656478881836,
      "step": 69
    },
    {
      "epoch": 0.14961261020571734,
      "grad_norm": 7.623144170728904,
      "learning_rate": 6.948331786284207e-07,
      "logits/chosen": -1.0306965112686157,
      "logits/rejected": -1.0405011177062988,
      "logps/chosen": -489.2743835449219,
      "logps/rejected": -580.343994140625,
      "loss": 0.5627,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.306447982788086,
      "rewards/margins": 0.7942487001419067,
      "rewards/rejected": -3.100696563720703,
      "step": 70
    },
    {
      "epoch": 0.15174993320865615,
      "grad_norm": 7.340408773148345,
      "learning_rate": 6.943753560095204e-07,
      "logits/chosen": -1.1813827753067017,
      "logits/rejected": -1.1872248649597168,
      "logps/chosen": -516.26953125,
      "logps/rejected": -557.9083251953125,
      "loss": 0.563,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.189997434616089,
      "rewards/margins": 0.49108394980430603,
      "rewards/rejected": -2.681081533432007,
      "step": 71
    },
    {
      "epoch": 0.15388725621159496,
      "grad_norm": 9.09254480811679,
      "learning_rate": 6.938982656316509e-07,
      "logits/chosen": -1.108089566230774,
      "logits/rejected": -1.1164076328277588,
      "logps/chosen": -483.02099609375,
      "logps/rejected": -499.59503173828125,
      "loss": 0.5837,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.988714575767517,
      "rewards/margins": 0.3150157928466797,
      "rewards/rejected": -2.3037304878234863,
      "step": 72
    },
    {
      "epoch": 0.1560245792145338,
      "grad_norm": 10.656479693855966,
      "learning_rate": 6.934019341879603e-07,
      "logits/chosen": -0.9998635649681091,
      "logits/rejected": -0.9844257831573486,
      "logps/chosen": -449.13592529296875,
      "logps/rejected": -543.6675415039062,
      "loss": 0.6031,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9461640119552612,
      "rewards/margins": 0.7130802869796753,
      "rewards/rejected": -2.6592442989349365,
      "step": 73
    },
    {
      "epoch": 0.15816190221747262,
      "grad_norm": 10.135891828278513,
      "learning_rate": 6.928863894481325e-07,
      "logits/chosen": -1.057843804359436,
      "logits/rejected": -1.0019515752792358,
      "logps/chosen": -499.46343994140625,
      "logps/rejected": -607.9090576171875,
      "loss": 0.6289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.925887942314148,
      "rewards/margins": 0.7596069574356079,
      "rewards/rejected": -2.685494899749756,
      "step": 74
    },
    {
      "epoch": 0.16029922522041143,
      "grad_norm": 11.196785279167383,
      "learning_rate": 6.923516602568319e-07,
      "logits/chosen": -1.10731840133667,
      "logits/rejected": -1.1392724514007568,
      "logps/chosen": -572.486083984375,
      "logps/rejected": -581.3994140625,
      "loss": 0.6307,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.843285322189331,
      "rewards/margins": 0.3052109479904175,
      "rewards/rejected": -3.148496150970459,
      "step": 75
    },
    {
      "epoch": 0.16243654822335024,
      "grad_norm": 8.037071469258775,
      "learning_rate": 6.917977765320918e-07,
      "logits/chosen": -1.0746040344238281,
      "logits/rejected": -1.0597764253616333,
      "logps/chosen": -504.6460266113281,
      "logps/rejected": -606.507080078125,
      "loss": 0.5222,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3728506565093994,
      "rewards/margins": 0.8532505035400391,
      "rewards/rejected": -3.2261009216308594,
      "step": 76
    },
    {
      "epoch": 0.16457387122628908,
      "grad_norm": 8.046205826147432,
      "learning_rate": 6.912247692636382e-07,
      "logits/chosen": -0.9819946885108948,
      "logits/rejected": -0.9038519859313965,
      "logps/chosen": -402.05523681640625,
      "logps/rejected": -466.16973876953125,
      "loss": 0.5378,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7473161220550537,
      "rewards/margins": 0.5481730699539185,
      "rewards/rejected": -2.2954893112182617,
      "step": 77
    },
    {
      "epoch": 0.1667111942292279,
      "grad_norm": 8.442110303754333,
      "learning_rate": 6.90632670511158e-07,
      "logits/chosen": -1.0492621660232544,
      "logits/rejected": -1.0028660297393799,
      "logps/chosen": -452.12823486328125,
      "logps/rejected": -512.2908935546875,
      "loss": 0.5626,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.2684264183044434,
      "rewards/margins": 0.5885436534881592,
      "rewards/rejected": -2.8569703102111816,
      "step": 78
    },
    {
      "epoch": 0.1688485172321667,
      "grad_norm": 8.834706933620803,
      "learning_rate": 6.900215134025036e-07,
      "logits/chosen": -0.9072580933570862,
      "logits/rejected": -0.896135687828064,
      "logps/chosen": -446.088134765625,
      "logps/rejected": -527.0875854492188,
      "loss": 0.5476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0337717533111572,
      "rewards/margins": 0.5865947604179382,
      "rewards/rejected": -2.6203665733337402,
      "step": 79
    },
    {
      "epoch": 0.17098584023510552,
      "grad_norm": 8.951583471345572,
      "learning_rate": 6.893913321318404e-07,
      "logits/chosen": -0.9730253219604492,
      "logits/rejected": -0.988685131072998,
      "logps/chosen": -460.3742370605469,
      "logps/rejected": -495.2950134277344,
      "loss": 0.5833,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2925660610198975,
      "rewards/margins": 0.45126643776893616,
      "rewards/rejected": -2.743832588195801,
      "step": 80
    },
    {
      "epoch": 0.17312316323804436,
      "grad_norm": 10.31156332894422,
      "learning_rate": 6.887421619577333e-07,
      "logits/chosen": -0.9978621602058411,
      "logits/rejected": -0.914361298084259,
      "logps/chosen": -577.339111328125,
      "logps/rejected": -620.412353515625,
      "loss": 0.5333,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5646235942840576,
      "rewards/margins": 0.5104097127914429,
      "rewards/rejected": -3.075033187866211,
      "step": 81
    },
    {
      "epoch": 0.17526048624098317,
      "grad_norm": 8.242407615983574,
      "learning_rate": 6.880740392011738e-07,
      "logits/chosen": -1.0842255353927612,
      "logits/rejected": -1.0887690782546997,
      "logps/chosen": -484.07733154296875,
      "logps/rejected": -524.2720947265625,
      "loss": 0.5808,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.4766807556152344,
      "rewards/margins": 0.48678654432296753,
      "rewards/rejected": -2.963467597961426,
      "step": 82
    },
    {
      "epoch": 0.17739780924392198,
      "grad_norm": 9.401026195070148,
      "learning_rate": 6.873870012435486e-07,
      "logits/chosen": -1.0653420686721802,
      "logits/rejected": -1.0636709928512573,
      "logps/chosen": -476.5955810546875,
      "logps/rejected": -574.3685302734375,
      "loss": 0.5926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0014445781707764,
      "rewards/margins": 1.0705955028533936,
      "rewards/rejected": -3.072040319442749,
      "step": 83
    },
    {
      "epoch": 0.1795351322468608,
      "grad_norm": 9.603090833181401,
      "learning_rate": 6.866810865245471e-07,
      "logits/chosen": -1.1408371925354004,
      "logits/rejected": -1.2083356380462646,
      "logps/chosen": -481.21270751953125,
      "logps/rejected": -499.5158386230469,
      "loss": 0.5634,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.340351104736328,
      "rewards/margins": 0.2937456965446472,
      "rewards/rejected": -2.634096622467041,
      "step": 84
    },
    {
      "epoch": 0.18167245524979964,
      "grad_norm": 8.394856753842978,
      "learning_rate": 6.859563345400111e-07,
      "logits/chosen": -1.0628814697265625,
      "logits/rejected": -0.9136342406272888,
      "logps/chosen": -399.5640869140625,
      "logps/rejected": -533.1271362304688,
      "loss": 0.5817,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9623302221298218,
      "rewards/margins": 1.1472116708755493,
      "rewards/rejected": -3.109541893005371,
      "step": 85
    },
    {
      "epoch": 0.18380977825273845,
      "grad_norm": 8.518782849174428,
      "learning_rate": 6.852127858397256e-07,
      "logits/chosen": -1.066598892211914,
      "logits/rejected": -1.0352439880371094,
      "logps/chosen": -561.7534790039062,
      "logps/rejected": -711.9557495117188,
      "loss": 0.521,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.871816396713257,
      "rewards/margins": 1.3357863426208496,
      "rewards/rejected": -4.207602500915527,
      "step": 86
    },
    {
      "epoch": 0.18594710125567726,
      "grad_norm": 9.531847125052328,
      "learning_rate": 6.844504820251493e-07,
      "logits/chosen": -0.8711273670196533,
      "logits/rejected": -0.7599645853042603,
      "logps/chosen": -630.6957397460938,
      "logps/rejected": -768.0391845703125,
      "loss": 0.5164,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.074097156524658,
      "rewards/margins": 1.3850178718566895,
      "rewards/rejected": -4.459115028381348,
      "step": 87
    },
    {
      "epoch": 0.18808442425861607,
      "grad_norm": 11.193362121063247,
      "learning_rate": 6.836694657470869e-07,
      "logits/chosen": -1.0319950580596924,
      "logits/rejected": -0.9569318294525146,
      "logps/chosen": -283.17431640625,
      "logps/rejected": -333.98797607421875,
      "loss": 0.5972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.3249003887176514,
      "rewards/margins": 0.47719448804855347,
      "rewards/rejected": -1.8020949363708496,
      "step": 88
    },
    {
      "epoch": 0.1902217472615549,
      "grad_norm": 12.5393990232581,
      "learning_rate": 6.828697807033038e-07,
      "logits/chosen": -1.016239881515503,
      "logits/rejected": -0.9772812724113464,
      "logps/chosen": -387.52056884765625,
      "logps/rejected": -477.847412109375,
      "loss": 0.5869,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.845961332321167,
      "rewards/margins": 0.6933265328407288,
      "rewards/rejected": -2.539287805557251,
      "step": 89
    },
    {
      "epoch": 0.19235907026449373,
      "grad_norm": 11.082808377296098,
      "learning_rate": 6.820514716360799e-07,
      "logits/chosen": -0.9989445209503174,
      "logits/rejected": -1.0149664878845215,
      "logps/chosen": -550.3873291015625,
      "logps/rejected": -627.486328125,
      "loss": 0.5262,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.6533279418945312,
      "rewards/margins": 0.7726477980613708,
      "rewards/rejected": -3.425976037979126,
      "step": 90
    },
    {
      "epoch": 0.19449639326743254,
      "grad_norm": 10.136450815566235,
      "learning_rate": 6.812145843297075e-07,
      "logits/chosen": -0.9787087440490723,
      "logits/rejected": -0.9385868310928345,
      "logps/chosen": -541.9707641601562,
      "logps/rejected": -582.03759765625,
      "loss": 0.537,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.860710382461548,
      "rewards/margins": 0.4204621911048889,
      "rewards/rejected": -3.281172513961792,
      "step": 91
    },
    {
      "epoch": 0.19663371627037135,
      "grad_norm": 9.876929179683522,
      "learning_rate": 6.803591656079286e-07,
      "logits/chosen": -1.1552644968032837,
      "logits/rejected": -1.1622107028961182,
      "logps/chosen": -502.1416015625,
      "logps/rejected": -571.0797729492188,
      "loss": 0.5405,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.547774076461792,
      "rewards/margins": 0.6767189502716064,
      "rewards/rejected": -3.2244930267333984,
      "step": 92
    },
    {
      "epoch": 0.1987710392733102,
      "grad_norm": 8.350168280012312,
      "learning_rate": 6.794852633313161e-07,
      "logits/chosen": -0.8592644333839417,
      "logits/rejected": -0.8664505481719971,
      "logps/chosen": -385.8682556152344,
      "logps/rejected": -391.97564697265625,
      "loss": 0.5449,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.7356358766555786,
      "rewards/margins": 0.2472929060459137,
      "rewards/rejected": -1.9829288721084595,
      "step": 93
    },
    {
      "epoch": 0.200908362276249,
      "grad_norm": 10.381869975048419,
      "learning_rate": 6.785929263945951e-07,
      "logits/chosen": -0.9588985443115234,
      "logits/rejected": -0.982033371925354,
      "logps/chosen": -562.7421875,
      "logps/rejected": -626.1223754882812,
      "loss": 0.5568,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.8523409366607666,
      "rewards/margins": 0.6437615752220154,
      "rewards/rejected": -3.4961023330688477,
      "step": 94
    },
    {
      "epoch": 0.20304568527918782,
      "grad_norm": 8.994516081218443,
      "learning_rate": 6.776822047239079e-07,
      "logits/chosen": -1.0020171403884888,
      "logits/rejected": -0.9751790761947632,
      "logps/chosen": -417.7941589355469,
      "logps/rejected": -482.18511962890625,
      "loss": 0.5187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.281278610229492,
      "rewards/margins": 0.5854904651641846,
      "rewards/rejected": -2.866769313812256,
      "step": 95
    },
    {
      "epoch": 0.20518300828212663,
      "grad_norm": 9.433844383462462,
      "learning_rate": 6.767531492740206e-07,
      "logits/chosen": -1.1744908094406128,
      "logits/rejected": -1.152523159980774,
      "logps/chosen": -517.0827026367188,
      "logps/rejected": -538.0679931640625,
      "loss": 0.5032,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9695494174957275,
      "rewards/margins": 0.5735199451446533,
      "rewards/rejected": -2.543069362640381,
      "step": 96
    },
    {
      "epoch": 0.20732033128506547,
      "grad_norm": 12.358671729964394,
      "learning_rate": 6.758058120254714e-07,
      "logits/chosen": -1.0569499731063843,
      "logits/rejected": -0.9990131855010986,
      "logps/chosen": -499.7625427246094,
      "logps/rejected": -545.9909057617188,
      "loss": 0.5433,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.38741397857666,
      "rewards/margins": 0.40309280157089233,
      "rewards/rejected": -2.7905068397521973,
      "step": 97
    },
    {
      "epoch": 0.20945765428800428,
      "grad_norm": 13.359908029714509,
      "learning_rate": 6.748402459816636e-07,
      "logits/chosen": -0.9123142957687378,
      "logits/rejected": -0.9710780382156372,
      "logps/chosen": -639.8603515625,
      "logps/rejected": -638.2657470703125,
      "loss": 0.5369,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.3181374073028564,
      "rewards/margins": 0.10219268500804901,
      "rewards/rejected": -3.420330286026001,
      "step": 98
    },
    {
      "epoch": 0.2115949772909431,
      "grad_norm": 9.360941415209796,
      "learning_rate": 6.738565051658988e-07,
      "logits/chosen": -0.9268811941146851,
      "logits/rejected": -1.0143418312072754,
      "logps/chosen": -504.1163330078125,
      "logps/rejected": -515.728271484375,
      "loss": 0.5485,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.5163679122924805,
      "rewards/margins": 0.32084643840789795,
      "rewards/rejected": -2.837214469909668,
      "step": 99
    },
    {
      "epoch": 0.2137323002938819,
      "grad_norm": 10.094138733652892,
      "learning_rate": 6.728546446183552e-07,
      "logits/chosen": -0.6902378797531128,
      "logits/rejected": -0.7625846266746521,
      "logps/chosen": -370.24859619140625,
      "logps/rejected": -434.644287109375,
      "loss": 0.5813,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.854517936706543,
      "rewards/margins": 0.39516037702560425,
      "rewards/rejected": -2.249678373336792,
      "step": 100
    },
    {
      "epoch": 0.2137323002938819,
      "eval_logits/chosen": -1.1082046031951904,
      "eval_logits/rejected": -1.1243932247161865,
      "eval_logps/chosen": -541.2933349609375,
      "eval_logps/rejected": -612.4234008789062,
      "eval_loss": 0.5237933397293091,
      "eval_rewards/accuracies": 0.7298387289047241,
      "eval_rewards/chosen": -2.681633710861206,
      "eval_rewards/margins": 0.7722625732421875,
      "eval_rewards/rejected": -3.4538962841033936,
      "eval_runtime": 141.8383,
      "eval_samples_per_second": 13.826,
      "eval_steps_per_second": 0.437,
      "step": 100
    },
    {
      "epoch": 0.21586962329682075,
      "grad_norm": 13.595024692594054,
      "learning_rate": 6.718347203930077e-07,
      "logits/chosen": -1.0973420143127441,
      "logits/rejected": -1.1126675605773926,
      "logps/chosen": -547.3043823242188,
      "logps/rejected": -647.8222045898438,
      "loss": 0.6014,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.594785690307617,
      "rewards/margins": 1.0374321937561035,
      "rewards/rejected": -3.6322176456451416,
      "step": 101
    },
    {
      "epoch": 0.21800694629975956,
      "grad_norm": 10.896734874060659,
      "learning_rate": 6.707967895544916e-07,
      "logits/chosen": -1.0957213640213013,
      "logits/rejected": -0.958062469959259,
      "logps/chosen": -466.9610595703125,
      "logps/rejected": -579.3587036132812,
      "loss": 0.4572,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.131477117538452,
      "rewards/margins": 0.9194568395614624,
      "rewards/rejected": -3.050933837890625,
      "step": 102
    },
    {
      "epoch": 0.22014426930269837,
      "grad_norm": 7.969619734080228,
      "learning_rate": 6.697409101749103e-07,
      "logits/chosen": -0.9213029146194458,
      "logits/rejected": -0.8857940435409546,
      "logps/chosen": -506.6609191894531,
      "logps/rejected": -597.8681640625,
      "loss": 0.4681,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.1986255645751953,
      "rewards/margins": 0.973738431930542,
      "rewards/rejected": -3.172363758087158,
      "step": 103
    },
    {
      "epoch": 0.22228159230563718,
      "grad_norm": 8.749688164160379,
      "learning_rate": 6.686671413305856e-07,
      "logits/chosen": -1.1210659742355347,
      "logits/rejected": -1.0568785667419434,
      "logps/chosen": -527.4649658203125,
      "logps/rejected": -666.8826904296875,
      "loss": 0.4962,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.895803213119507,
      "rewards/margins": 1.2766165733337402,
      "rewards/rejected": -4.172420024871826,
      "step": 104
    },
    {
      "epoch": 0.224418915308576,
      "grad_norm": 10.816785495540321,
      "learning_rate": 6.675755430987527e-07,
      "logits/chosen": -1.098203420639038,
      "logits/rejected": -1.043796181678772,
      "logps/chosen": -406.22186279296875,
      "logps/rejected": -515.1035766601562,
      "loss": 0.5235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0460591316223145,
      "rewards/margins": 0.9124148488044739,
      "rewards/rejected": -2.9584736824035645,
      "step": 105
    },
    {
      "epoch": 0.22655623831151483,
      "grad_norm": 16.006357368038902,
      "learning_rate": 6.66466176554199e-07,
      "logits/chosen": -1.1074966192245483,
      "logits/rejected": -1.1250437498092651,
      "logps/chosen": -511.63519287109375,
      "logps/rejected": -583.53759765625,
      "loss": 0.6286,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.4097325801849365,
      "rewards/margins": 0.7932775020599365,
      "rewards/rejected": -3.203009843826294,
      "step": 106
    },
    {
      "epoch": 0.22869356131445365,
      "grad_norm": 10.145999208049496,
      "learning_rate": 6.653391037658466e-07,
      "logits/chosen": -0.8575969934463501,
      "logits/rejected": -0.8596129417419434,
      "logps/chosen": -600.901123046875,
      "logps/rejected": -702.7192993164062,
      "loss": 0.4988,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.2228050231933594,
      "rewards/margins": 0.8592959642410278,
      "rewards/rejected": -4.082100868225098,
      "step": 107
    },
    {
      "epoch": 0.23083088431739246,
      "grad_norm": 9.90205874296923,
      "learning_rate": 6.641943877932801e-07,
      "logits/chosen": -0.982778787612915,
      "logits/rejected": -0.967957615852356,
      "logps/chosen": -637.15478515625,
      "logps/rejected": -736.35546875,
      "loss": 0.5006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.472803831100464,
      "rewards/margins": 0.9627398252487183,
      "rewards/rejected": -4.435543537139893,
      "step": 108
    },
    {
      "epoch": 0.23296820732033127,
      "grad_norm": 10.737852627360994,
      "learning_rate": 6.630320926832178e-07,
      "logits/chosen": -1.0073411464691162,
      "logits/rejected": -0.9530993700027466,
      "logps/chosen": -543.8530883789062,
      "logps/rejected": -669.4638061523438,
      "loss": 0.5518,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.8368093967437744,
      "rewards/margins": 1.1880838871002197,
      "rewards/rejected": -4.024893283843994,
      "step": 109
    },
    {
      "epoch": 0.2351055303232701,
      "grad_norm": 10.25518470798023,
      "learning_rate": 6.618522834659287e-07,
      "logits/chosen": -1.0670475959777832,
      "logits/rejected": -0.9968966245651245,
      "logps/chosen": -551.1478271484375,
      "logps/rejected": -710.794921875,
      "loss": 0.4774,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1101412773132324,
      "rewards/margins": 1.3935075998306274,
      "rewards/rejected": -4.50364875793457,
      "step": 110
    },
    {
      "epoch": 0.23724285332620892,
      "grad_norm": 10.88392855187359,
      "learning_rate": 6.606550261515939e-07,
      "logits/chosen": -1.1226112842559814,
      "logits/rejected": -1.1211527585983276,
      "logps/chosen": -496.4976806640625,
      "logps/rejected": -598.9983520507812,
      "loss": 0.5214,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4094114303588867,
      "rewards/margins": 0.967483639717102,
      "rewards/rejected": -3.37689471244812,
      "step": 111
    },
    {
      "epoch": 0.23938017632914774,
      "grad_norm": 14.5461852961629,
      "learning_rate": 6.594403877266134e-07,
      "logits/chosen": -0.9625245332717896,
      "logits/rejected": -1.0075706243515015,
      "logps/chosen": -479.2593688964844,
      "logps/rejected": -569.9885864257812,
      "loss": 0.5683,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.5545759201049805,
      "rewards/margins": 0.8655453324317932,
      "rewards/rejected": -3.420121431350708,
      "step": 112
    },
    {
      "epoch": 0.24151749933208655,
      "grad_norm": 10.257155762835067,
      "learning_rate": 6.582084361498582e-07,
      "logits/chosen": -1.0782970190048218,
      "logits/rejected": -1.0272691249847412,
      "logps/chosen": -431.1832580566406,
      "logps/rejected": -507.42401123046875,
      "loss": 0.4517,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3007423877716064,
      "rewards/margins": 0.6344060301780701,
      "rewards/rejected": -2.935148239135742,
      "step": 113
    },
    {
      "epoch": 0.2436548223350254,
      "grad_norm": 12.281013811232151,
      "learning_rate": 6.569592403488683e-07,
      "logits/chosen": -0.8818793296813965,
      "logits/rejected": -0.8034133315086365,
      "logps/chosen": -575.656005859375,
      "logps/rejected": -665.9652709960938,
      "loss": 0.5607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9418675899505615,
      "rewards/margins": 0.8054768443107605,
      "rewards/rejected": -3.747344493865967,
      "step": 114
    },
    {
      "epoch": 0.2457921453379642,
      "grad_norm": 12.328703519510336,
      "learning_rate": 6.556928702159954e-07,
      "logits/chosen": -1.0138773918151855,
      "logits/rejected": -0.9439175128936768,
      "logps/chosen": -389.8624267578125,
      "logps/rejected": -458.929443359375,
      "loss": 0.5415,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.944897174835205,
      "rewards/margins": 0.5071231126785278,
      "rewards/rejected": -2.4520201683044434,
      "step": 115
    },
    {
      "epoch": 0.24792946834090301,
      "grad_norm": 11.019958842452935,
      "learning_rate": 6.544093966044933e-07,
      "logits/chosen": -1.0471012592315674,
      "logits/rejected": -1.046525001525879,
      "logps/chosen": -589.4481811523438,
      "logps/rejected": -666.8422241210938,
      "loss": 0.4744,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.920729160308838,
      "rewards/margins": 0.9037714004516602,
      "rewards/rejected": -3.824500560760498,
      "step": 116
    },
    {
      "epoch": 0.25006679134384185,
      "grad_norm": 12.320353833685262,
      "learning_rate": 6.531088913245536e-07,
      "logits/chosen": -0.9775707721710205,
      "logits/rejected": -1.0437175035476685,
      "logps/chosen": -559.2511596679688,
      "logps/rejected": -629.9223022460938,
      "loss": 0.5403,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.863795280456543,
      "rewards/margins": 0.9548764228820801,
      "rewards/rejected": -3.818671703338623,
      "step": 117
    },
    {
      "epoch": 0.25220411434678064,
      "grad_norm": 13.023997265472861,
      "learning_rate": 6.517914271392868e-07,
      "logits/chosen": -0.9753481149673462,
      "logits/rejected": -0.9616096019744873,
      "logps/chosen": -393.8183288574219,
      "logps/rejected": -467.1902770996094,
      "loss": 0.5117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0589523315429688,
      "rewards/margins": 0.6962214708328247,
      "rewards/rejected": -2.755174160003662,
      "step": 118
    },
    {
      "epoch": 0.2543414373497195,
      "grad_norm": 10.577094903989462,
      "learning_rate": 6.504570777606531e-07,
      "logits/chosen": -1.1235861778259277,
      "logits/rejected": -1.0339062213897705,
      "logps/chosen": -567.0230712890625,
      "logps/rejected": -711.5052490234375,
      "loss": 0.5027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.0050158500671387,
      "rewards/margins": 1.1521648168563843,
      "rewards/rejected": -4.1571807861328125,
      "step": 119
    },
    {
      "epoch": 0.2564787603526583,
      "grad_norm": 10.49693020043088,
      "learning_rate": 6.491059178453366e-07,
      "logits/chosen": -0.9402109980583191,
      "logits/rejected": -0.9911918044090271,
      "logps/chosen": -524.8615112304688,
      "logps/rejected": -612.2509765625,
      "loss": 0.4693,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.6408395767211914,
      "rewards/margins": 0.9583368301391602,
      "rewards/rejected": -3.5991761684417725,
      "step": 120
    },
    {
      "epoch": 0.2586160833555971,
      "grad_norm": 11.423558750572097,
      "learning_rate": 6.477380229905688e-07,
      "logits/chosen": -1.072409987449646,
      "logits/rejected": -1.1194437742233276,
      "logps/chosen": -561.1153564453125,
      "logps/rejected": -709.1156616210938,
      "loss": 0.4927,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.7978341579437256,
      "rewards/margins": 1.542366862297058,
      "rewards/rejected": -4.340201377868652,
      "step": 121
    },
    {
      "epoch": 0.26075340635853594,
      "grad_norm": 11.084946490602835,
      "learning_rate": 6.463534697298995e-07,
      "logits/chosen": -0.9931835532188416,
      "logits/rejected": -0.8609973788261414,
      "logps/chosen": -422.6210632324219,
      "logps/rejected": -529.1079711914062,
      "loss": 0.4607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9958739280700684,
      "rewards/margins": 0.8684347867965698,
      "rewards/rejected": -2.8643088340759277,
      "step": 122
    },
    {
      "epoch": 0.26289072936147473,
      "grad_norm": 12.662380910929564,
      "learning_rate": 6.449523355289137e-07,
      "logits/chosen": -1.0383917093276978,
      "logits/rejected": -1.0384039878845215,
      "logps/chosen": -552.278076171875,
      "logps/rejected": -629.6370849609375,
      "loss": 0.5599,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.6315407752990723,
      "rewards/margins": 0.8207972049713135,
      "rewards/rejected": -3.4523377418518066,
      "step": 123
    },
    {
      "epoch": 0.26502805236441357,
      "grad_norm": 12.221280650614627,
      "learning_rate": 6.435346987808983e-07,
      "logits/chosen": -1.0482319593429565,
      "logits/rejected": -1.0103380680084229,
      "logps/chosen": -566.80078125,
      "logps/rejected": -658.3344116210938,
      "loss": 0.5172,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.72255539894104,
      "rewards/margins": 1.0078556537628174,
      "rewards/rejected": -3.7304112911224365,
      "step": 124
    },
    {
      "epoch": 0.2671653753673524,
      "grad_norm": 10.864584700405295,
      "learning_rate": 6.421006388024559e-07,
      "logits/chosen": -0.9935693740844727,
      "logits/rejected": -1.0035309791564941,
      "logps/chosen": -450.3325500488281,
      "logps/rejected": -518.1874389648438,
      "loss": 0.4811,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.3707103729248047,
      "rewards/margins": 0.6181319952011108,
      "rewards/rejected": -2.988842248916626,
      "step": 125
    },
    {
      "epoch": 0.2693026983702912,
      "grad_norm": 12.045454242902174,
      "learning_rate": 6.406502358290663e-07,
      "logits/chosen": -1.1015965938568115,
      "logits/rejected": -1.08380126953125,
      "logps/chosen": -496.258056640625,
      "logps/rejected": -653.9327392578125,
      "loss": 0.5494,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.305727481842041,
      "rewards/margins": 1.4904249906539917,
      "rewards/rejected": -3.796152353286743,
      "step": 126
    },
    {
      "epoch": 0.27144002137323003,
      "grad_norm": 18.364703541342568,
      "learning_rate": 6.391835710105981e-07,
      "logits/chosen": -0.941745400428772,
      "logits/rejected": -0.9719420671463013,
      "logps/chosen": -511.3363952636719,
      "logps/rejected": -570.1834716796875,
      "loss": 0.5391,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.4952189922332764,
      "rewards/margins": 0.8485791683197021,
      "rewards/rejected": -3.3437979221343994,
      "step": 127
    },
    {
      "epoch": 0.2735773443761689,
      "grad_norm": 11.918550043243753,
      "learning_rate": 6.377007264067684e-07,
      "logits/chosen": -0.9891614317893982,
      "logits/rejected": -0.9654213190078735,
      "logps/chosen": -455.00775146484375,
      "logps/rejected": -501.5223083496094,
      "loss": 0.5492,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.259676218032837,
      "rewards/margins": 0.5743985176086426,
      "rewards/rejected": -2.8340747356414795,
      "step": 128
    },
    {
      "epoch": 0.27571466737910766,
      "grad_norm": 15.016960436290145,
      "learning_rate": 6.36201784982551e-07,
      "logits/chosen": -0.9479376673698425,
      "logits/rejected": -1.0105921030044556,
      "logps/chosen": -484.4723815917969,
      "logps/rejected": -579.9881591796875,
      "loss": 0.4852,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.237053871154785,
      "rewards/margins": 1.103590488433838,
      "rewards/rejected": -3.340644121170044,
      "step": 129
    },
    {
      "epoch": 0.2778519903820465,
      "grad_norm": 10.336460981894195,
      "learning_rate": 6.346868306035347e-07,
      "logits/chosen": -1.0786710977554321,
      "logits/rejected": -1.0136293172836304,
      "logps/chosen": -531.4271850585938,
      "logps/rejected": -632.714599609375,
      "loss": 0.4997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.405646562576294,
      "rewards/margins": 1.1792259216308594,
      "rewards/rejected": -3.5848724842071533,
      "step": 130
    },
    {
      "epoch": 0.2799893133849853,
      "grad_norm": 13.583887221626354,
      "learning_rate": 6.331559480312316e-07,
      "logits/chosen": -0.8798739910125732,
      "logits/rejected": -0.9477823972702026,
      "logps/chosen": -596.0357055664062,
      "logps/rejected": -668.8670654296875,
      "loss": 0.4991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.9465270042419434,
      "rewards/margins": 0.8481062054634094,
      "rewards/rejected": -3.794633150100708,
      "step": 131
    },
    {
      "epoch": 0.2821266363879241,
      "grad_norm": 14.394323156381558,
      "learning_rate": 6.316092229183339e-07,
      "logits/chosen": -0.9489572048187256,
      "logits/rejected": -0.9146859645843506,
      "logps/chosen": -600.4107666015625,
      "logps/rejected": -661.3778076171875,
      "loss": 0.5529,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.2759668827056885,
      "rewards/margins": 0.545818567276001,
      "rewards/rejected": -3.8217854499816895,
      "step": 132
    },
    {
      "epoch": 0.28426395939086296,
      "grad_norm": 11.385386930454487,
      "learning_rate": 6.300467418039222e-07,
      "logits/chosen": -0.960631787776947,
      "logits/rejected": -0.8502722382545471,
      "logps/chosen": -396.5357360839844,
      "logps/rejected": -541.553955078125,
      "loss": 0.5093,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8977466821670532,
      "rewards/margins": 1.2623937129974365,
      "rewards/rejected": -3.1601402759552,
      "step": 133
    },
    {
      "epoch": 0.28640128239380175,
      "grad_norm": 11.957264036760781,
      "learning_rate": 6.284685921086233e-07,
      "logits/chosen": -0.8640112280845642,
      "logits/rejected": -0.8963266611099243,
      "logps/chosen": -384.329345703125,
      "logps/rejected": -407.7789611816406,
      "loss": 0.498,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.0386128425598145,
      "rewards/margins": 0.3072957992553711,
      "rewards/rejected": -2.3459086418151855,
      "step": 134
    },
    {
      "epoch": 0.2885386053967406,
      "grad_norm": 11.759955525639521,
      "learning_rate": 6.268748621297195e-07,
      "logits/chosen": -0.8902631998062134,
      "logits/rejected": -0.7775343656539917,
      "logps/chosen": -403.541015625,
      "logps/rejected": -534.9651489257812,
      "loss": 0.4668,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9903576374053955,
      "rewards/margins": 1.1432865858078003,
      "rewards/rejected": -3.1336441040039062,
      "step": 135
    },
    {
      "epoch": 0.29067592839967943,
      "grad_norm": 11.950216444117117,
      "learning_rate": 6.252656410362074e-07,
      "logits/chosen": -0.9792582988739014,
      "logits/rejected": -0.9614965915679932,
      "logps/chosen": -561.1026611328125,
      "logps/rejected": -663.7088012695312,
      "loss": 0.4924,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.82108998298645,
      "rewards/margins": 1.011682391166687,
      "rewards/rejected": -3.8327722549438477,
      "step": 136
    },
    {
      "epoch": 0.2928132514026182,
      "grad_norm": 10.772908788991131,
      "learning_rate": 6.236410188638104e-07,
      "logits/chosen": -1.053274393081665,
      "logits/rejected": -0.9625363349914551,
      "logps/chosen": -572.0177001953125,
      "logps/rejected": -789.4164428710938,
      "loss": 0.5162,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.667423725128174,
      "rewards/margins": 2.069328546524048,
      "rewards/rejected": -4.736752510070801,
      "step": 137
    },
    {
      "epoch": 0.29495057440555705,
      "grad_norm": 9.953041872331532,
      "learning_rate": 6.220010865099398e-07,
      "logits/chosen": -1.0631476640701294,
      "logits/rejected": -1.0828768014907837,
      "logps/chosen": -395.8948669433594,
      "logps/rejected": -478.61468505859375,
      "loss": 0.4991,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0671029090881348,
      "rewards/margins": 0.7652881145477295,
      "rewards/rejected": -2.8323915004730225,
      "step": 138
    },
    {
      "epoch": 0.29708789740849584,
      "grad_norm": 9.56597037699992,
      "learning_rate": 6.203459357286098e-07,
      "logits/chosen": -1.0143028497695923,
      "logits/rejected": -1.0154125690460205,
      "logps/chosen": -517.6961669921875,
      "logps/rejected": -596.7298583984375,
      "loss": 0.4613,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.6147890090942383,
      "rewards/margins": 0.7214634418487549,
      "rewards/rejected": -3.336252212524414,
      "step": 139
    },
    {
      "epoch": 0.2992252204114347,
      "grad_norm": 13.08709351718772,
      "learning_rate": 6.186756591253038e-07,
      "logits/chosen": -0.9290235042572021,
      "logits/rejected": -0.9178426861763,
      "logps/chosen": -527.4471435546875,
      "logps/rejected": -568.3038330078125,
      "loss": 0.5523,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.790428638458252,
      "rewards/margins": 0.6073124408721924,
      "rewards/rejected": -3.3977413177490234,
      "step": 140
    },
    {
      "epoch": 0.3013625434143735,
      "grad_norm": 9.930492612600617,
      "learning_rate": 6.169903501517932e-07,
      "logits/chosen": -0.9746488928794861,
      "logits/rejected": -0.9574599862098694,
      "logps/chosen": -426.73760986328125,
      "logps/rejected": -535.6727294921875,
      "loss": 0.5198,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.180694580078125,
      "rewards/margins": 0.977644145488739,
      "rewards/rejected": -3.158338785171509,
      "step": 141
    },
    {
      "epoch": 0.3034998664173123,
      "grad_norm": 14.86115549874432,
      "learning_rate": 6.152901031009086e-07,
      "logits/chosen": -0.9366909265518188,
      "logits/rejected": -0.9158231019973755,
      "logps/chosen": -489.0951232910156,
      "logps/rejected": -574.0645141601562,
      "loss": 0.5373,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.401134967803955,
      "rewards/margins": 0.8239090442657471,
      "rewards/rejected": -3.2250442504882812,
      "step": 142
    },
    {
      "epoch": 0.30563718942025114,
      "grad_norm": 18.2841162585259,
      "learning_rate": 6.135750131012639e-07,
      "logits/chosen": -0.9252594709396362,
      "logits/rejected": -0.8823320269584656,
      "logps/chosen": -500.0283203125,
      "logps/rejected": -582.243408203125,
      "loss": 0.5163,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.57209849357605,
      "rewards/margins": 0.7344849109649658,
      "rewards/rejected": -3.3065831661224365,
      "step": 143
    },
    {
      "epoch": 0.3077745124231899,
      "grad_norm": 10.361990112128243,
      "learning_rate": 6.118451761119344e-07,
      "logits/chosen": -0.9340752363204956,
      "logits/rejected": -0.9739060997962952,
      "logps/chosen": -686.3743286132812,
      "logps/rejected": -827.3367919921875,
      "loss": 0.4274,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.355973243713379,
      "rewards/margins": 1.5688568353652954,
      "rewards/rejected": -4.924829483032227,
      "step": 144
    },
    {
      "epoch": 0.30991183542612877,
      "grad_norm": 14.742711073001722,
      "learning_rate": 6.101006889170879e-07,
      "logits/chosen": -0.9780830144882202,
      "logits/rejected": -0.9924584031105042,
      "logps/chosen": -611.8880615234375,
      "logps/rejected": -783.9960327148438,
      "loss": 0.4892,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.101553440093994,
      "rewards/margins": 1.6389245986938477,
      "rewards/rejected": -4.740478038787842,
      "step": 145
    },
    {
      "epoch": 0.3120491584290676,
      "grad_norm": 11.760151720162954,
      "learning_rate": 6.083416491205693e-07,
      "logits/chosen": -0.9051653146743774,
      "logits/rejected": -0.9137300252914429,
      "logps/chosen": -518.42333984375,
      "logps/rejected": -624.302490234375,
      "loss": 0.4934,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.439613103866577,
      "rewards/margins": 1.1922130584716797,
      "rewards/rejected": -3.631826162338257,
      "step": 146
    },
    {
      "epoch": 0.3141864814320064,
      "grad_norm": 12.767953140987824,
      "learning_rate": 6.065681551404392e-07,
      "logits/chosen": -0.9767911434173584,
      "logits/rejected": -0.9007346034049988,
      "logps/chosen": -498.14361572265625,
      "logps/rejected": -551.3754272460938,
      "loss": 0.5575,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.6738038063049316,
      "rewards/margins": 0.7255844473838806,
      "rewards/rejected": -3.399388313293457,
      "step": 147
    },
    {
      "epoch": 0.31632380443494523,
      "grad_norm": 11.792817714413513,
      "learning_rate": 6.047803062034688e-07,
      "logits/chosen": -1.0714683532714844,
      "logits/rejected": -1.0007489919662476,
      "logps/chosen": -524.3717041015625,
      "logps/rejected": -721.374267578125,
      "loss": 0.5315,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.6513381004333496,
      "rewards/margins": 1.8740673065185547,
      "rewards/rejected": -4.525405406951904,
      "step": 148
    },
    {
      "epoch": 0.3184611274378841,
      "grad_norm": 10.394135670198176,
      "learning_rate": 6.029782023395871e-07,
      "logits/chosen": -0.99624103307724,
      "logits/rejected": -0.970421314239502,
      "logps/chosen": -520.40771484375,
      "logps/rejected": -551.2412109375,
      "loss": 0.4518,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.831399917602539,
      "rewards/margins": 0.5486774444580078,
      "rewards/rejected": -3.380077362060547,
      "step": 149
    },
    {
      "epoch": 0.32059845044082286,
      "grad_norm": 14.89234343098954,
      "learning_rate": 6.011619443762841e-07,
      "logits/chosen": -1.1248900890350342,
      "logits/rejected": -1.1022447347640991,
      "logps/chosen": -546.7406005859375,
      "logps/rejected": -674.46484375,
      "loss": 0.5543,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.7904787063598633,
      "rewards/margins": 1.1329127550125122,
      "rewards/rejected": -3.923391342163086,
      "step": 150
    },
    {
      "epoch": 0.3227357734437617,
      "grad_norm": 10.760811605793574,
      "learning_rate": 5.993316339329707e-07,
      "logits/chosen": -0.8464213013648987,
      "logits/rejected": -0.9315951466560364,
      "logps/chosen": -557.2722778320312,
      "logps/rejected": -562.7290649414062,
      "loss": 0.5073,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.0413103103637695,
      "rewards/margins": 0.24244722723960876,
      "rewards/rejected": -3.2837576866149902,
      "step": 151
    },
    {
      "epoch": 0.3248730964467005,
      "grad_norm": 12.468671998929876,
      "learning_rate": 5.974873734152916e-07,
      "logits/chosen": -0.9382638931274414,
      "logits/rejected": -0.8903573751449585,
      "logps/chosen": -517.4348754882812,
      "logps/rejected": -586.1632690429688,
      "loss": 0.559,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.7158446311950684,
      "rewards/margins": 0.5773513317108154,
      "rewards/rejected": -3.293196201324463,
      "step": 152
    },
    {
      "epoch": 0.3270104194496393,
      "grad_norm": 10.71297529597072,
      "learning_rate": 5.95629266009397e-07,
      "logits/chosen": -0.9615129232406616,
      "logits/rejected": -0.9363632202148438,
      "logps/chosen": -530.434814453125,
      "logps/rejected": -629.813720703125,
      "loss": 0.4965,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.732804775238037,
      "rewards/margins": 1.224104881286621,
      "rewards/rejected": -3.9569101333618164,
      "step": 153
    },
    {
      "epoch": 0.32914774245257816,
      "grad_norm": 12.327388201247391,
      "learning_rate": 5.937574156761684e-07,
      "logits/chosen": -0.9693203568458557,
      "logits/rejected": -0.9414520263671875,
      "logps/chosen": -467.2398681640625,
      "logps/rejected": -549.6279296875,
      "loss": 0.558,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.324397087097168,
      "rewards/margins": 0.6942296624183655,
      "rewards/rejected": -3.018627166748047,
      "step": 154
    },
    {
      "epoch": 0.33128506545551695,
      "grad_norm": 11.452079247996043,
      "learning_rate": 5.918719271454026e-07,
      "logits/chosen": -0.9070168137550354,
      "logits/rejected": -0.9717684984207153,
      "logps/chosen": -463.07073974609375,
      "logps/rejected": -531.9006958007812,
      "loss": 0.4757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.246826171875,
      "rewards/margins": 0.8463835120201111,
      "rewards/rejected": -3.0932095050811768,
      "step": 155
    },
    {
      "epoch": 0.3334223884584558,
      "grad_norm": 10.387010221236869,
      "learning_rate": 5.89972905909952e-07,
      "logits/chosen": -0.9737594127655029,
      "logits/rejected": -0.9816542267799377,
      "logps/chosen": -482.228515625,
      "logps/rejected": -550.7467041015625,
      "loss": 0.4944,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.8164095878601074,
      "rewards/margins": 0.6258185505867004,
      "rewards/rejected": -3.442227840423584,
      "step": 156
    },
    {
      "epoch": 0.3355597114613946,
      "grad_norm": 13.81426757522365,
      "learning_rate": 5.880604582198217e-07,
      "logits/chosen": -0.8456093072891235,
      "logits/rejected": -0.831813395023346,
      "logps/chosen": -598.4921875,
      "logps/rejected": -722.5546875,
      "loss": 0.4857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.0666327476501465,
      "rewards/margins": 1.3944388628005981,
      "rewards/rejected": -4.461071014404297,
      "step": 157
    },
    {
      "epoch": 0.3376970344643334,
      "grad_norm": 12.739583339970373,
      "learning_rate": 5.86134691076226e-07,
      "logits/chosen": -0.9977485537528992,
      "logits/rejected": -0.9227933287620544,
      "logps/chosen": -458.33758544921875,
      "logps/rejected": -590.2880249023438,
      "loss": 0.4951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.321625232696533,
      "rewards/margins": 1.1777639389038086,
      "rewards/rejected": -3.499389410018921,
      "step": 158
    },
    {
      "epoch": 0.33983435746727225,
      "grad_norm": 11.40592684455961,
      "learning_rate": 5.841957122256004e-07,
      "logits/chosen": -0.9870405793190002,
      "logits/rejected": -0.9605780839920044,
      "logps/chosen": -460.610107421875,
      "logps/rejected": -543.5918579101562,
      "loss": 0.4798,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.591230869293213,
      "rewards/margins": 0.8468169569969177,
      "rewards/rejected": -3.4380481243133545,
      "step": 159
    },
    {
      "epoch": 0.34197168047021104,
      "grad_norm": 11.54298974939068,
      "learning_rate": 5.822436301535739e-07,
      "logits/chosen": -0.9439342617988586,
      "logits/rejected": -0.896612823009491,
      "logps/chosen": -549.1138916015625,
      "logps/rejected": -611.7896728515625,
      "loss": 0.4867,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.6459498405456543,
      "rewards/margins": 0.6619236469268799,
      "rewards/rejected": -3.307873487472534,
      "step": 160
    },
    {
      "epoch": 0.3441090034731499,
      "grad_norm": 10.028108962774086,
      "learning_rate": 5.802785540788994e-07,
      "logits/chosen": -1.018284797668457,
      "logits/rejected": -0.9662111401557922,
      "logps/chosen": -477.1954650878906,
      "logps/rejected": -642.691650390625,
      "loss": 0.4616,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.5217745304107666,
      "rewards/margins": 1.391313910484314,
      "rewards/rejected": -3.913088321685791,
      "step": 161
    },
    {
      "epoch": 0.3462463264760887,
      "grad_norm": 10.240941578340253,
      "learning_rate": 5.783005939473424e-07,
      "logits/chosen": -0.8555217981338501,
      "logits/rejected": -0.8488653898239136,
      "logps/chosen": -490.9669189453125,
      "logps/rejected": -589.8624267578125,
      "loss": 0.474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.722303628921509,
      "rewards/margins": 0.9005026817321777,
      "rewards/rejected": -3.6228065490722656,
      "step": 162
    },
    {
      "epoch": 0.3483836494790275,
      "grad_norm": 11.409205514657348,
      "learning_rate": 5.763098604255298e-07,
      "logits/chosen": -0.8463901281356812,
      "logits/rejected": -0.8225279450416565,
      "logps/chosen": -470.57440185546875,
      "logps/rejected": -639.9716796875,
      "loss": 0.4536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5086305141448975,
      "rewards/margins": 1.750978946685791,
      "rewards/rejected": -4.259609222412109,
      "step": 163
    },
    {
      "epoch": 0.35052097248196634,
      "grad_norm": 10.927823097709238,
      "learning_rate": 5.743064648947584e-07,
      "logits/chosen": -0.7492021322250366,
      "logits/rejected": -0.7506992816925049,
      "logps/chosen": -561.3836669921875,
      "logps/rejected": -710.4789428710938,
      "loss": 0.5088,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.9847238063812256,
      "rewards/margins": 1.428779125213623,
      "rewards/rejected": -4.413503170013428,
      "step": 164
    },
    {
      "epoch": 0.3526582954849052,
      "grad_norm": 13.549577455647025,
      "learning_rate": 5.722905194447625e-07,
      "logits/chosen": -1.0247427225112915,
      "logits/rejected": -1.0630146265029907,
      "logps/chosen": -696.59228515625,
      "logps/rejected": -761.5941772460938,
      "loss": 0.5028,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.713867425918579,
      "rewards/margins": 0.7951303124427795,
      "rewards/rejected": -4.508997440338135,
      "step": 165
    },
    {
      "epoch": 0.35479561848784397,
      "grad_norm": 15.326243524128424,
      "learning_rate": 5.702621368674431e-07,
      "logits/chosen": -0.9843580722808838,
      "logits/rejected": -0.9505161643028259,
      "logps/chosen": -439.31622314453125,
      "logps/rejected": -539.7349243164062,
      "loss": 0.5054,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5671231746673584,
      "rewards/margins": 0.8157074451446533,
      "rewards/rejected": -3.3828306198120117,
      "step": 166
    },
    {
      "epoch": 0.3569329414907828,
      "grad_norm": 13.81282026576703,
      "learning_rate": 5.682214306505567e-07,
      "logits/chosen": -0.7752354145050049,
      "logits/rejected": -0.831868052482605,
      "logps/chosen": -618.5186157226562,
      "logps/rejected": -734.19580078125,
      "loss": 0.4584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.667295217514038,
      "rewards/margins": 1.2412769794464111,
      "rewards/rejected": -4.908572196960449,
      "step": 167
    },
    {
      "epoch": 0.3590702644937216,
      "grad_norm": 12.331430233086655,
      "learning_rate": 5.661685149713663e-07,
      "logits/chosen": -0.8932394981384277,
      "logits/rejected": -0.8569494485855103,
      "logps/chosen": -504.1289367675781,
      "logps/rejected": -628.0838012695312,
      "loss": 0.5174,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.809536933898926,
      "rewards/margins": 1.0781981945037842,
      "rewards/rejected": -3.88773512840271,
      "step": 168
    },
    {
      "epoch": 0.36120758749666043,
      "grad_norm": 13.419219251759696,
      "learning_rate": 5.641035046902522e-07,
      "logits/chosen": -0.8772855997085571,
      "logits/rejected": -0.8467257618904114,
      "logps/chosen": -470.7606201171875,
      "logps/rejected": -565.2396240234375,
      "loss": 0.4684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.8519928455352783,
      "rewards/margins": 0.8433955907821655,
      "rewards/rejected": -3.695388078689575,
      "step": 169
    },
    {
      "epoch": 0.36334491049959927,
      "grad_norm": 12.695684401858434,
      "learning_rate": 5.620265153442869e-07,
      "logits/chosen": -1.0130642652511597,
      "logits/rejected": -0.9847390055656433,
      "logps/chosen": -514.5957641601562,
      "logps/rejected": -585.168212890625,
      "loss": 0.469,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.0407803058624268,
      "rewards/margins": 0.6757410168647766,
      "rewards/rejected": -3.7165212631225586,
      "step": 170
    },
    {
      "epoch": 0.36548223350253806,
      "grad_norm": 12.147780619595629,
      "learning_rate": 5.599376631407699e-07,
      "logits/chosen": -0.9069080948829651,
      "logits/rejected": -0.8009949922561646,
      "logps/chosen": -547.1576538085938,
      "logps/rejected": -679.2531127929688,
      "loss": 0.4299,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.197631359100342,
      "rewards/margins": 1.0914721488952637,
      "rewards/rejected": -4.2891035079956055,
      "step": 171
    },
    {
      "epoch": 0.3676195565054769,
      "grad_norm": 18.51274657202258,
      "learning_rate": 5.578370649507255e-07,
      "logits/chosen": -0.6547280550003052,
      "logits/rejected": -0.6987937092781067,
      "logps/chosen": -577.7064819335938,
      "logps/rejected": -660.9352416992188,
      "loss": 0.5682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2265751361846924,
      "rewards/margins": 0.9371775388717651,
      "rewards/rejected": -4.163752555847168,
      "step": 172
    },
    {
      "epoch": 0.36975687950841574,
      "grad_norm": 13.62299944344872,
      "learning_rate": 5.557248383023656e-07,
      "logits/chosen": -0.9801725745201111,
      "logits/rejected": -0.945483386516571,
      "logps/chosen": -449.10614013671875,
      "logps/rejected": -567.4400634765625,
      "loss": 0.4765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.63840389251709,
      "rewards/margins": 1.1649121046066284,
      "rewards/rejected": -3.803316116333008,
      "step": 173
    },
    {
      "epoch": 0.3718942025113545,
      "grad_norm": 12.16862925856691,
      "learning_rate": 5.536011013745121e-07,
      "logits/chosen": -0.9143638610839844,
      "logits/rejected": -0.9123052358627319,
      "logps/chosen": -483.41357421875,
      "logps/rejected": -624.1827392578125,
      "loss": 0.459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.4780359268188477,
      "rewards/margins": 1.3084805011749268,
      "rewards/rejected": -3.7865164279937744,
      "step": 174
    },
    {
      "epoch": 0.37403152551429336,
      "grad_norm": 11.507324455166394,
      "learning_rate": 5.514659729899857e-07,
      "logits/chosen": -0.7952810525894165,
      "logits/rejected": -0.7528566718101501,
      "logps/chosen": -498.6630554199219,
      "logps/rejected": -630.5216064453125,
      "loss": 0.4045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.8591041564941406,
      "rewards/margins": 1.1144087314605713,
      "rewards/rejected": -3.973512649536133,
      "step": 175
    },
    {
      "epoch": 0.37616884851723215,
      "grad_norm": 12.170272129622308,
      "learning_rate": 5.493195726089582e-07,
      "logits/chosen": -0.9865118265151978,
      "logits/rejected": -0.9716078639030457,
      "logps/chosen": -543.14501953125,
      "logps/rejected": -683.6224365234375,
      "loss": 0.4348,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.054544448852539,
      "rewards/margins": 1.5414537191390991,
      "rewards/rejected": -4.595998287200928,
      "step": 176
    },
    {
      "epoch": 0.378306171520171,
      "grad_norm": 13.22726269166873,
      "learning_rate": 5.471620203222677e-07,
      "logits/chosen": -0.9932882189750671,
      "logits/rejected": -0.991521418094635,
      "logps/chosen": -507.20574951171875,
      "logps/rejected": -601.6077880859375,
      "loss": 0.4227,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.048675060272217,
      "rewards/margins": 0.8585546016693115,
      "rewards/rejected": -3.9072299003601074,
      "step": 177
    },
    {
      "epoch": 0.3804434945231098,
      "grad_norm": 11.607213817333788,
      "learning_rate": 5.449934368447002e-07,
      "logits/chosen": -0.8857173919677734,
      "logits/rejected": -0.9733960628509521,
      "logps/chosen": -627.8464965820312,
      "logps/rejected": -713.438720703125,
      "loss": 0.4544,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.520371437072754,
      "rewards/margins": 1.112282156944275,
      "rewards/rejected": -4.63265323638916,
      "step": 178
    },
    {
      "epoch": 0.3825808175260486,
      "grad_norm": 12.188016147935972,
      "learning_rate": 5.428139435082358e-07,
      "logits/chosen": -0.7584793567657471,
      "logits/rejected": -0.8010637760162354,
      "logps/chosen": -566.6948852539062,
      "logps/rejected": -713.6017456054688,
      "loss": 0.4106,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4136064052581787,
      "rewards/margins": 1.416290044784546,
      "rewards/rejected": -4.829895973205566,
      "step": 179
    },
    {
      "epoch": 0.38471814052898745,
      "grad_norm": 13.803380819286584,
      "learning_rate": 5.406236622552594e-07,
      "logits/chosen": -0.8117521405220032,
      "logits/rejected": -0.8442290425300598,
      "logps/chosen": -658.14990234375,
      "logps/rejected": -791.3058471679688,
      "loss": 0.4795,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9962399005889893,
      "rewards/margins": 1.3741449117660522,
      "rewards/rejected": -5.370384693145752,
      "step": 180
    },
    {
      "epoch": 0.38685546353192624,
      "grad_norm": 19.74453585010364,
      "learning_rate": 5.384227156317388e-07,
      "logits/chosen": -0.9625721573829651,
      "logits/rejected": -1.0246632099151611,
      "logps/chosen": -570.2412109375,
      "logps/rejected": -627.0347900390625,
      "loss": 0.5302,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.3639683723449707,
      "rewards/margins": 0.4717191457748413,
      "rewards/rejected": -3.8356876373291016,
      "step": 181
    },
    {
      "epoch": 0.3889927865348651,
      "grad_norm": 12.180825522167304,
      "learning_rate": 5.362112267803678e-07,
      "logits/chosen": -1.1118032932281494,
      "logits/rejected": -1.0548309087753296,
      "logps/chosen": -561.20654296875,
      "logps/rejected": -801.3348388671875,
      "loss": 0.407,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0669779777526855,
      "rewards/margins": 2.297924041748047,
      "rewards/rejected": -5.364901542663574,
      "step": 182
    },
    {
      "epoch": 0.3911301095378039,
      "grad_norm": 15.493196005683158,
      "learning_rate": 5.339893194336765e-07,
      "logits/chosen": -0.9748885035514832,
      "logits/rejected": -0.9212169647216797,
      "logps/chosen": -556.37451171875,
      "logps/rejected": -747.4850463867188,
      "loss": 0.434,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.1176629066467285,
      "rewards/margins": 2.0257248878479004,
      "rewards/rejected": -5.143387794494629,
      "step": 183
    },
    {
      "epoch": 0.3932674325407427,
      "grad_norm": 12.834279479584271,
      "learning_rate": 5.317571179071092e-07,
      "logits/chosen": -1.0676939487457275,
      "logits/rejected": -1.0940359830856323,
      "logps/chosen": -547.4573364257812,
      "logps/rejected": -625.5507202148438,
      "loss": 0.4441,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.1368393898010254,
      "rewards/margins": 0.7552158236503601,
      "rewards/rejected": -3.8920555114746094,
      "step": 184
    },
    {
      "epoch": 0.39540475554368154,
      "grad_norm": 15.858497539317051,
      "learning_rate": 5.295147470920672e-07,
      "logits/chosen": -0.8825098276138306,
      "logits/rejected": -0.8697827458381653,
      "logps/chosen": -606.1354370117188,
      "logps/rejected": -752.2478637695312,
      "loss": 0.4948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.499366521835327,
      "rewards/margins": 1.4578311443328857,
      "rewards/rejected": -4.957198143005371,
      "step": 185
    },
    {
      "epoch": 0.3975420785466204,
      "grad_norm": 15.81390161765314,
      "learning_rate": 5.27262332448923e-07,
      "logits/chosen": -0.9162149429321289,
      "logits/rejected": -0.8807965517044067,
      "logps/chosen": -588.79736328125,
      "logps/rejected": -775.908447265625,
      "loss": 0.4836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.620396852493286,
      "rewards/margins": 1.820716142654419,
      "rewards/rejected": -5.441113471984863,
      "step": 186
    },
    {
      "epoch": 0.39967940154955917,
      "grad_norm": 11.930553509668082,
      "learning_rate": 5.25e-07,
      "logits/chosen": -0.878287136554718,
      "logits/rejected": -0.8447045683860779,
      "logps/chosen": -588.80908203125,
      "logps/rejected": -710.580078125,
      "loss": 0.4217,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.770730495452881,
      "rewards/margins": 1.0582211017608643,
      "rewards/rejected": -4.828950881958008,
      "step": 187
    },
    {
      "epoch": 0.401816724552498,
      "grad_norm": 19.78691767583468,
      "learning_rate": 5.227278763225213e-07,
      "logits/chosen": -1.002894639968872,
      "logits/rejected": -0.9999670386314392,
      "logps/chosen": -729.7056884765625,
      "logps/rejected": -895.4324340820312,
      "loss": 0.488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.301171779632568,
      "rewards/margins": 1.7632997035980225,
      "rewards/rejected": -6.06447172164917,
      "step": 188
    },
    {
      "epoch": 0.4039540475554368,
      "grad_norm": 15.17666246755368,
      "learning_rate": 5.204460885415285e-07,
      "logits/chosen": -0.9192298650741577,
      "logits/rejected": -0.9201673269271851,
      "logps/chosen": -612.3890380859375,
      "logps/rejected": -722.1416625976562,
      "loss": 0.4896,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5543270111083984,
      "rewards/margins": 1.225655198097229,
      "rewards/rejected": -4.779982089996338,
      "step": 189
    },
    {
      "epoch": 0.40609137055837563,
      "grad_norm": 12.423293515766808,
      "learning_rate": 5.181547643227682e-07,
      "logits/chosen": -0.8379946947097778,
      "logits/rejected": -0.8106329441070557,
      "logps/chosen": -546.194580078125,
      "logps/rejected": -636.456298828125,
      "loss": 0.4722,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.497601270675659,
      "rewards/margins": 0.9037814736366272,
      "rewards/rejected": -4.4013824462890625,
      "step": 190
    },
    {
      "epoch": 0.40822869356131447,
      "grad_norm": 16.79590350569075,
      "learning_rate": 5.158540318655495e-07,
      "logits/chosen": -1.0028809309005737,
      "logits/rejected": -0.9536342024803162,
      "logps/chosen": -669.4102783203125,
      "logps/rejected": -854.4193115234375,
      "loss": 0.4629,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.044215679168701,
      "rewards/margins": 1.6450707912445068,
      "rewards/rejected": -5.689286708831787,
      "step": 191
    },
    {
      "epoch": 0.41036601656425326,
      "grad_norm": 14.164634270897476,
      "learning_rate": 5.135440198955717e-07,
      "logits/chosen": -0.97246915102005,
      "logits/rejected": -0.9198447465896606,
      "logps/chosen": -513.80859375,
      "logps/rejected": -543.1536865234375,
      "loss": 0.4983,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.1685631275177,
      "rewards/margins": 0.3603837192058563,
      "rewards/rejected": -3.528946876525879,
      "step": 192
    },
    {
      "epoch": 0.4125033395671921,
      "grad_norm": 15.329408000607371,
      "learning_rate": 5.112248576577213e-07,
      "logits/chosen": -0.7749034762382507,
      "logits/rejected": -0.8009266257286072,
      "logps/chosen": -596.5591430664062,
      "logps/rejected": -797.4959716796875,
      "loss": 0.4978,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.661137342453003,
      "rewards/margins": 1.7927237749099731,
      "rewards/rejected": -5.453861236572266,
      "step": 193
    },
    {
      "epoch": 0.41464066257013094,
      "grad_norm": 14.042005201292891,
      "learning_rate": 5.088966749088414e-07,
      "logits/chosen": -1.116222858428955,
      "logits/rejected": -1.090318202972412,
      "logps/chosen": -603.7730102539062,
      "logps/rejected": -657.4857177734375,
      "loss": 0.4895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.191464424133301,
      "rewards/margins": 0.7639944553375244,
      "rewards/rejected": -3.9554591178894043,
      "step": 194
    },
    {
      "epoch": 0.4167779855730697,
      "grad_norm": 13.675394100746447,
      "learning_rate": 5.065596019104713e-07,
      "logits/chosen": -1.0994431972503662,
      "logits/rejected": -1.091101884841919,
      "logps/chosen": -700.6111450195312,
      "logps/rejected": -847.7877197265625,
      "loss": 0.4107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.003596305847168,
      "rewards/margins": 1.4772356748580933,
      "rewards/rejected": -5.480831623077393,
      "step": 195
    },
    {
      "epoch": 0.41891530857600856,
      "grad_norm": 18.55076809192975,
      "learning_rate": 5.042137694215592e-07,
      "logits/chosen": -0.9493328928947449,
      "logits/rejected": -1.022355079650879,
      "logps/chosen": -641.4091796875,
      "logps/rejected": -696.9602661132812,
      "loss": 0.618,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.780533790588379,
      "rewards/margins": 0.6779485940933228,
      "rewards/rejected": -4.458482265472412,
      "step": 196
    },
    {
      "epoch": 0.42105263157894735,
      "grad_norm": 12.163973418481676,
      "learning_rate": 5.018593086911453e-07,
      "logits/chosen": -1.0403311252593994,
      "logits/rejected": -1.069019079208374,
      "logps/chosen": -671.8824462890625,
      "logps/rejected": -849.7545166015625,
      "loss": 0.4197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.7160799503326416,
      "rewards/margins": 1.7244200706481934,
      "rewards/rejected": -5.440499782562256,
      "step": 197
    },
    {
      "epoch": 0.4231899545818862,
      "grad_norm": 13.736070000914967,
      "learning_rate": 4.994963514510188e-07,
      "logits/chosen": -1.0466636419296265,
      "logits/rejected": -0.9742124676704407,
      "logps/chosen": -536.3780517578125,
      "logps/rejected": -622.6585083007812,
      "loss": 0.4457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.165748119354248,
      "rewards/margins": 0.7600272297859192,
      "rewards/rejected": -3.9257755279541016,
      "step": 198
    },
    {
      "epoch": 0.425327277584825,
      "grad_norm": 11.871725745717265,
      "learning_rate": 4.971250299083479e-07,
      "logits/chosen": -0.9257810711860657,
      "logits/rejected": -0.9252554774284363,
      "logps/chosen": -556.8322143554688,
      "logps/rejected": -656.6981201171875,
      "loss": 0.4155,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.365008592605591,
      "rewards/margins": 1.075837254524231,
      "rewards/rejected": -4.440845966339111,
      "step": 199
    },
    {
      "epoch": 0.4274646005877638,
      "grad_norm": 12.50996465065088,
      "learning_rate": 4.947454767382822e-07,
      "logits/chosen": -0.9889892339706421,
      "logits/rejected": -1.004003882408142,
      "logps/chosen": -499.806884765625,
      "logps/rejected": -617.0054321289062,
      "loss": 0.5021,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.9506893157958984,
      "rewards/margins": 1.0361154079437256,
      "rewards/rejected": -3.986804485321045,
      "step": 200
    },
    {
      "epoch": 0.4274646005877638,
      "eval_logits/chosen": -1.084425926208496,
      "eval_logits/rejected": -1.103520393371582,
      "eval_logps/chosen": -613.6640625,
      "eval_logps/rejected": -715.6145629882812,
      "eval_loss": 0.44827935099601746,
      "eval_rewards/accuracies": 0.8024193644523621,
      "eval_rewards/chosen": -3.4053404331207275,
      "eval_rewards/margins": 1.080467700958252,
      "eval_rewards/rejected": -4.4858078956604,
      "eval_runtime": 126.9937,
      "eval_samples_per_second": 15.442,
      "eval_steps_per_second": 0.488,
      "step": 200
    },
    {
      "epoch": 0.42960192359070265,
      "grad_norm": 13.246746821586836,
      "learning_rate": 4.923578250765301e-07,
      "logits/chosen": -1.0361019372940063,
      "logits/rejected": -1.0069564580917358,
      "logps/chosen": -630.4815063476562,
      "logps/rejected": -819.7576293945312,
      "loss": 0.4381,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.616086483001709,
      "rewards/margins": 1.9921762943267822,
      "rewards/rejected": -5.608262538909912,
      "step": 201
    },
    {
      "epoch": 0.4317392465936415,
      "grad_norm": 13.986462198608523,
      "learning_rate": 4.899622085119092e-07,
      "logits/chosen": -1.0016858577728271,
      "logits/rejected": -1.0090727806091309,
      "logps/chosen": -639.8945922851562,
      "logps/rejected": -797.4194946289062,
      "loss": 0.389,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.3241751194000244,
      "rewards/margins": 1.6805753707885742,
      "rewards/rejected": -5.0047502517700195,
      "step": 202
    },
    {
      "epoch": 0.4338765695965803,
      "grad_norm": 12.938400299110988,
      "learning_rate": 4.875587610788733e-07,
      "logits/chosen": -1.0168877840042114,
      "logits/rejected": -1.0365827083587646,
      "logps/chosen": -593.1552124023438,
      "logps/rejected": -752.401611328125,
      "loss": 0.4125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.574855327606201,
      "rewards/margins": 1.3029221296310425,
      "rewards/rejected": -4.877778053283691,
      "step": 203
    },
    {
      "epoch": 0.4360138925995191,
      "grad_norm": 12.350696519100337,
      "learning_rate": 4.851476172500112e-07,
      "logits/chosen": -1.0002954006195068,
      "logits/rejected": -0.9592558145523071,
      "logps/chosen": -691.1764526367188,
      "logps/rejected": -894.4859619140625,
      "loss": 0.4075,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.358014106750488,
      "rewards/margins": 1.7022799253463745,
      "rewards/rejected": -6.060294151306152,
      "step": 204
    },
    {
      "epoch": 0.4381512156024579,
      "grad_norm": 14.780877866432451,
      "learning_rate": 4.827289119285249e-07,
      "logits/chosen": -0.9795655608177185,
      "logits/rejected": -0.9753063917160034,
      "logps/chosen": -560.5239868164062,
      "logps/rejected": -701.3786010742188,
      "loss": 0.466,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2604007720947266,
      "rewards/margins": 1.3381632566452026,
      "rewards/rejected": -4.5985636711120605,
      "step": 205
    },
    {
      "epoch": 0.44028853860539674,
      "grad_norm": 22.9673286565427,
      "learning_rate": 4.803027804406808e-07,
      "logits/chosen": -0.7228925228118896,
      "logits/rejected": -0.8121004700660706,
      "logps/chosen": -680.5167236328125,
      "logps/rejected": -703.938232421875,
      "loss": 0.559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.237366199493408,
      "rewards/margins": 0.45848214626312256,
      "rewards/rejected": -4.695847988128662,
      "step": 206
    },
    {
      "epoch": 0.4424258616083356,
      "grad_norm": 18.405605350322162,
      "learning_rate": 4.778693585282384e-07,
      "logits/chosen": -1.0253394842147827,
      "logits/rejected": -1.0438570976257324,
      "logps/chosen": -631.9534912109375,
      "logps/rejected": -705.13623046875,
      "loss": 0.5002,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.731057643890381,
      "rewards/margins": 0.8837456107139587,
      "rewards/rejected": -4.614803314208984,
      "step": 207
    },
    {
      "epoch": 0.44456318461127436,
      "grad_norm": 15.70410271681006,
      "learning_rate": 4.75428782340855e-07,
      "logits/chosen": -1.0454537868499756,
      "logits/rejected": -1.1345258951187134,
      "logps/chosen": -641.6697998046875,
      "logps/rejected": -731.5447387695312,
      "loss": 0.4413,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.9197850227355957,
      "rewards/margins": 0.858109712600708,
      "rewards/rejected": -4.777895450592041,
      "step": 208
    },
    {
      "epoch": 0.4467005076142132,
      "grad_norm": 13.037823833248444,
      "learning_rate": 4.7298118842846993e-07,
      "logits/chosen": -0.9263153076171875,
      "logits/rejected": -0.9316372871398926,
      "logps/chosen": -522.490966796875,
      "logps/rejected": -635.503173828125,
      "loss": 0.3848,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2573442459106445,
      "rewards/margins": 0.9754186272621155,
      "rewards/rejected": -4.232762336730957,
      "step": 209
    },
    {
      "epoch": 0.448837830617152,
      "grad_norm": 15.711193208519104,
      "learning_rate": 4.705267137336622e-07,
      "logits/chosen": -1.0621023178100586,
      "logits/rejected": -1.0041028261184692,
      "logps/chosen": -799.73974609375,
      "logps/rejected": -927.7545776367188,
      "loss": 0.4351,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.812376022338867,
      "rewards/margins": 1.1584749221801758,
      "rewards/rejected": -5.970850944519043,
      "step": 210
    },
    {
      "epoch": 0.45097515362009083,
      "grad_norm": 13.021464052671469,
      "learning_rate": 4.680654955839905e-07,
      "logits/chosen": -1.0352355241775513,
      "logits/rejected": -1.0864734649658203,
      "logps/chosen": -679.844970703125,
      "logps/rejected": -842.2390747070312,
      "loss": 0.3926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.970327138900757,
      "rewards/margins": 1.7980788946151733,
      "rewards/rejected": -5.768405914306641,
      "step": 211
    },
    {
      "epoch": 0.45311247662302967,
      "grad_norm": 13.400793392671966,
      "learning_rate": 4.655976716843085e-07,
      "logits/chosen": -0.9447561502456665,
      "logits/rejected": -0.9419984221458435,
      "logps/chosen": -577.1600341796875,
      "logps/rejected": -626.2113647460938,
      "loss": 0.4097,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.421278476715088,
      "rewards/margins": 0.7281789183616638,
      "rewards/rejected": -4.1494574546813965,
      "step": 212
    },
    {
      "epoch": 0.45524979962596845,
      "grad_norm": 15.76259483264769,
      "learning_rate": 4.63123380109061e-07,
      "logits/chosen": -0.9420887231826782,
      "logits/rejected": -0.9258626699447632,
      "logps/chosen": -604.66064453125,
      "logps/rejected": -688.6736450195312,
      "loss": 0.5374,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.727839708328247,
      "rewards/margins": 0.6107276082038879,
      "rewards/rejected": -4.33856725692749,
      "step": 213
    },
    {
      "epoch": 0.4573871226289073,
      "grad_norm": 14.47796429675634,
      "learning_rate": 4.6064275929455854e-07,
      "logits/chosen": -1.012191653251648,
      "logits/rejected": -0.9834191799163818,
      "logps/chosen": -539.5471801757812,
      "logps/rejected": -643.7244873046875,
      "loss": 0.3953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.355208396911621,
      "rewards/margins": 0.9154967665672302,
      "rewards/rejected": -4.270705223083496,
      "step": 214
    },
    {
      "epoch": 0.45952444563184613,
      "grad_norm": 18.127660409487774,
      "learning_rate": 4.581559480312316e-07,
      "logits/chosen": -0.9188220500946045,
      "logits/rejected": -0.8601971864700317,
      "logps/chosen": -562.9246826171875,
      "logps/rejected": -646.825439453125,
      "loss": 0.5093,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.4996066093444824,
      "rewards/margins": 0.7593184113502502,
      "rewards/rejected": -4.258925437927246,
      "step": 215
    },
    {
      "epoch": 0.4616617686347849,
      "grad_norm": 14.332440562045605,
      "learning_rate": 4.556630854558657e-07,
      "logits/chosen": -1.0821930170059204,
      "logits/rejected": -1.0690282583236694,
      "logps/chosen": -612.6143188476562,
      "logps/rejected": -785.2466430664062,
      "loss": 0.4259,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.40315580368042,
      "rewards/margins": 1.751339316368103,
      "rewards/rejected": -5.1544952392578125,
      "step": 216
    },
    {
      "epoch": 0.46379909163772376,
      "grad_norm": 13.6646330391651,
      "learning_rate": 4.5316431104381647e-07,
      "logits/chosen": -1.0587506294250488,
      "logits/rejected": -1.0372809171676636,
      "logps/chosen": -491.8074951171875,
      "logps/rejected": -638.18212890625,
      "loss": 0.4567,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.595409870147705,
      "rewards/margins": 1.3147836923599243,
      "rewards/rejected": -3.910193681716919,
      "step": 217
    },
    {
      "epoch": 0.46593641464066254,
      "grad_norm": 16.27324594458174,
      "learning_rate": 4.50659764601206e-07,
      "logits/chosen": -1.1704292297363281,
      "logits/rejected": -1.1407532691955566,
      "logps/chosen": -635.9962158203125,
      "logps/rejected": -709.2085571289062,
      "loss": 0.5169,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.4440693855285645,
      "rewards/margins": 0.8417525887489319,
      "rewards/rejected": -4.285822868347168,
      "step": 218
    },
    {
      "epoch": 0.4680737376436014,
      "grad_norm": 14.30405716244965,
      "learning_rate": 4.4814958625710095e-07,
      "logits/chosen": -0.9373888969421387,
      "logits/rejected": -0.9414705634117126,
      "logps/chosen": -641.3363037109375,
      "logps/rejected": -782.9466552734375,
      "loss": 0.4214,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.41910719871521,
      "rewards/margins": 1.5010746717453003,
      "rewards/rejected": -4.920182228088379,
      "step": 219
    },
    {
      "epoch": 0.4702110606465402,
      "grad_norm": 12.888240768536523,
      "learning_rate": 4.4563391645567203e-07,
      "logits/chosen": -0.9721742272377014,
      "logits/rejected": -0.9880850315093994,
      "logps/chosen": -563.5908813476562,
      "logps/rejected": -648.4769897460938,
      "loss": 0.4525,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.4122231006622314,
      "rewards/margins": 0.8691967725753784,
      "rewards/rejected": -4.28141975402832,
      "step": 220
    },
    {
      "epoch": 0.472348383649479,
      "grad_norm": 13.808905917996086,
      "learning_rate": 4.4311289594833626e-07,
      "logits/chosen": -1.062425136566162,
      "logits/rejected": -1.0666816234588623,
      "logps/chosen": -425.4259338378906,
      "logps/rejected": -508.6617736816406,
      "loss": 0.3815,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.534140110015869,
      "rewards/margins": 0.9357590079307556,
      "rewards/rejected": -3.4698996543884277,
      "step": 221
    },
    {
      "epoch": 0.47448570665241785,
      "grad_norm": 14.257159662099976,
      "learning_rate": 4.405866657858822e-07,
      "logits/chosen": -1.0133253335952759,
      "logits/rejected": -1.0463453531265259,
      "logps/chosen": -577.285888671875,
      "logps/rejected": -644.2965087890625,
      "loss": 0.4452,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.20715594291687,
      "rewards/margins": 0.7728344202041626,
      "rewards/rejected": -3.979990243911743,
      "step": 222
    },
    {
      "epoch": 0.4766230296553567,
      "grad_norm": 12.854510596268446,
      "learning_rate": 4.38055367310578e-07,
      "logits/chosen": -0.985044538974762,
      "logits/rejected": -0.9488839507102966,
      "logps/chosen": -645.5025024414062,
      "logps/rejected": -869.3623046875,
      "loss": 0.3977,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.6057004928588867,
      "rewards/margins": 1.8732173442840576,
      "rewards/rejected": -5.478918075561523,
      "step": 223
    },
    {
      "epoch": 0.4787603526582955,
      "grad_norm": 14.869442818796234,
      "learning_rate": 4.355191421482629e-07,
      "logits/chosen": -1.1076860427856445,
      "logits/rejected": -1.0764734745025635,
      "logps/chosen": -516.0100708007812,
      "logps/rejected": -644.3858642578125,
      "loss": 0.4573,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1032497882843018,
      "rewards/margins": 1.1473891735076904,
      "rewards/rejected": -4.250638961791992,
      "step": 224
    },
    {
      "epoch": 0.4808976756612343,
      "grad_norm": 16.43448798480314,
      "learning_rate": 4.329781322004241e-07,
      "logits/chosen": -1.0606333017349243,
      "logits/rejected": -1.0501140356063843,
      "logps/chosen": -663.1077880859375,
      "logps/rejected": -874.1226806640625,
      "loss": 0.4973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.6670713424682617,
      "rewards/margins": 1.8036597967147827,
      "rewards/rejected": -5.470730781555176,
      "step": 225
    },
    {
      "epoch": 0.4830349986641731,
      "grad_norm": 15.477278289904351,
      "learning_rate": 4.304324796362568e-07,
      "logits/chosen": -1.026381015777588,
      "logits/rejected": -0.9911444187164307,
      "logps/chosen": -616.499267578125,
      "logps/rejected": -757.9083862304688,
      "loss": 0.4483,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.533419609069824,
      "rewards/margins": 1.2801600694656372,
      "rewards/rejected": -4.81358003616333,
      "step": 226
    },
    {
      "epoch": 0.48517232166711194,
      "grad_norm": 14.181376472254154,
      "learning_rate": 4.2788232688471e-07,
      "logits/chosen": -1.0257827043533325,
      "logits/rejected": -1.030511498451233,
      "logps/chosen": -623.668212890625,
      "logps/rejected": -722.494873046875,
      "loss": 0.4315,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.7645962238311768,
      "rewards/margins": 1.0766507387161255,
      "rewards/rejected": -4.841247081756592,
      "step": 227
    },
    {
      "epoch": 0.4873096446700508,
      "grad_norm": 18.128852851106277,
      "learning_rate": 4.253278166265177e-07,
      "logits/chosen": -1.0358821153640747,
      "logits/rejected": -1.0768516063690186,
      "logps/chosen": -567.903564453125,
      "logps/rejected": -689.8753051757812,
      "loss": 0.4725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.21205997467041,
      "rewards/margins": 1.3463387489318848,
      "rewards/rejected": -4.558398723602295,
      "step": 228
    },
    {
      "epoch": 0.48944696767298956,
      "grad_norm": 13.499906597454904,
      "learning_rate": 4.2276909178621567e-07,
      "logits/chosen": -0.9607468843460083,
      "logits/rejected": -0.9298718571662903,
      "logps/chosen": -749.897705078125,
      "logps/rejected": -971.5405883789062,
      "loss": 0.3638,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.647284030914307,
      "rewards/margins": 2.187747001647949,
      "rewards/rejected": -6.835031032562256,
      "step": 229
    },
    {
      "epoch": 0.4915842906759284,
      "grad_norm": 15.151757518766736,
      "learning_rate": 4.2020629552414545e-07,
      "logits/chosen": -1.0237997770309448,
      "logits/rejected": -0.9573734998703003,
      "logps/chosen": -746.3142700195312,
      "logps/rejected": -845.5519409179688,
      "loss": 0.4511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.615874767303467,
      "rewards/margins": 1.053206205368042,
      "rewards/rejected": -5.669081211090088,
      "step": 230
    },
    {
      "epoch": 0.49372161367886724,
      "grad_norm": 15.54384712409316,
      "learning_rate": 4.176395712284436e-07,
      "logits/chosen": -1.0997838973999023,
      "logits/rejected": -1.0569286346435547,
      "logps/chosen": -650.4686279296875,
      "logps/rejected": -801.4287109375,
      "loss": 0.4244,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.111543655395508,
      "rewards/margins": 1.6189815998077393,
      "rewards/rejected": -5.730525970458984,
      "step": 231
    },
    {
      "epoch": 0.49585893668180603,
      "grad_norm": 20.212574699658393,
      "learning_rate": 4.1506906250702017e-07,
      "logits/chosen": -0.9895690083503723,
      "logits/rejected": -0.9893757700920105,
      "logps/chosen": -653.65673828125,
      "logps/rejected": -774.2328491210938,
      "loss": 0.4351,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.012841701507568,
      "rewards/margins": 1.2045834064483643,
      "rewards/rejected": -5.217425346374512,
      "step": 232
    },
    {
      "epoch": 0.49799625968474487,
      "grad_norm": 19.25316743472589,
      "learning_rate": 4.124949131795228e-07,
      "logits/chosen": -0.9950495362281799,
      "logits/rejected": -0.9298911690711975,
      "logps/chosen": -593.9339599609375,
      "logps/rejected": -793.3387451171875,
      "loss": 0.3809,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.7703654766082764,
      "rewards/margins": 1.754205346107483,
      "rewards/rejected": -5.524570941925049,
      "step": 233
    },
    {
      "epoch": 0.5001335826876837,
      "grad_norm": 18.738876938950654,
      "learning_rate": 4.0991726726929135e-07,
      "logits/chosen": -1.06260085105896,
      "logits/rejected": -1.1126683950424194,
      "logps/chosen": -576.23583984375,
      "logps/rejected": -658.715576171875,
      "loss": 0.5016,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.6776123046875,
      "rewards/margins": 0.8450465202331543,
      "rewards/rejected": -4.5226593017578125,
      "step": 234
    },
    {
      "epoch": 0.5022709056906225,
      "grad_norm": 15.19487228526955,
      "learning_rate": 4.073362689952982e-07,
      "logits/chosen": -0.895923912525177,
      "logits/rejected": -0.9245651364326477,
      "logps/chosen": -693.7532348632812,
      "logps/rejected": -714.86865234375,
      "loss": 0.3918,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -4.3402838706970215,
      "rewards/margins": 0.3496522903442383,
      "rewards/rejected": -4.68993616104126,
      "step": 235
    },
    {
      "epoch": 0.5044082286935613,
      "grad_norm": 23.63383518264876,
      "learning_rate": 4.047520627640808e-07,
      "logits/chosen": -1.0637860298156738,
      "logits/rejected": -1.0301114320755005,
      "logps/chosen": -648.5076293945312,
      "logps/rejected": -772.224609375,
      "loss": 0.4001,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.6958110332489014,
      "rewards/margins": 1.3828696012496948,
      "rewards/rejected": -5.078680515289307,
      "step": 236
    },
    {
      "epoch": 0.5065455516965002,
      "grad_norm": 26.41816558524286,
      "learning_rate": 4.02164793161661e-07,
      "logits/chosen": -0.9920711517333984,
      "logits/rejected": -0.9205961227416992,
      "logps/chosen": -606.1458129882812,
      "logps/rejected": -882.5693969726562,
      "loss": 0.4818,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.58052396774292,
      "rewards/margins": 0.41025519371032715,
      "rewards/rejected": -3.9907796382904053,
      "step": 237
    },
    {
      "epoch": 0.508682874699439,
      "grad_norm": 15.74897212135201,
      "learning_rate": 3.9957460494545616e-07,
      "logits/chosen": -0.9549577236175537,
      "logits/rejected": -0.9211697578430176,
      "logps/chosen": -669.394287109375,
      "logps/rejected": -709.4468994140625,
      "loss": 0.4783,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.0179338455200195,
      "rewards/margins": 0.4471871554851532,
      "rewards/rejected": -4.465120792388916,
      "step": 238
    },
    {
      "epoch": 0.5108201977023777,
      "grad_norm": 16.016701961339525,
      "learning_rate": 3.969816430361794e-07,
      "logits/chosen": -1.0792049169540405,
      "logits/rejected": -0.9834136366844177,
      "logps/chosen": -673.969970703125,
      "logps/rejected": -964.8361206054688,
      "loss": 0.3911,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.139585018157959,
      "rewards/margins": 2.691437244415283,
      "rewards/rejected": -6.831022262573242,
      "step": 239
    },
    {
      "epoch": 0.5129575207053166,
      "grad_norm": 17.928342726480253,
      "learning_rate": 3.9438605250973193e-07,
      "logits/chosen": -0.9689183831214905,
      "logits/rejected": -0.9091028571128845,
      "logps/chosen": -529.1758422851562,
      "logps/rejected": -694.4168090820312,
      "loss": 0.4055,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.4839565753936768,
      "rewards/margins": 1.5238869190216064,
      "rewards/rejected": -5.007843494415283,
      "step": 240
    },
    {
      "epoch": 0.5150948437082554,
      "grad_norm": 13.751574315942849,
      "learning_rate": 3.9178797858908544e-07,
      "logits/chosen": -0.9920436143875122,
      "logits/rejected": -0.9840397834777832,
      "logps/chosen": -706.0386352539062,
      "logps/rejected": -849.04345703125,
      "loss": 0.3468,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.169567108154297,
      "rewards/margins": 1.2529165744781494,
      "rewards/rejected": -5.422484397888184,
      "step": 241
    },
    {
      "epoch": 0.5172321667111942,
      "grad_norm": 13.92465051011581,
      "learning_rate": 3.891875666361577e-07,
      "logits/chosen": -0.9852272868156433,
      "logits/rejected": -0.9285619258880615,
      "logps/chosen": -515.2431030273438,
      "logps/rejected": -713.544189453125,
      "loss": 0.3992,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.9760236740112305,
      "rewards/margins": 1.73358952999115,
      "rewards/rejected": -4.709612846374512,
      "step": 242
    },
    {
      "epoch": 0.5193694897141331,
      "grad_norm": 15.781523786771364,
      "learning_rate": 3.865849621436787e-07,
      "logits/chosen": -1.0415364503860474,
      "logits/rejected": -1.0579416751861572,
      "logps/chosen": -614.417236328125,
      "logps/rejected": -755.8591918945312,
      "loss": 0.4447,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.9967360496520996,
      "rewards/margins": 1.3200428485870361,
      "rewards/rejected": -5.316779136657715,
      "step": 243
    },
    {
      "epoch": 0.5215068127170719,
      "grad_norm": 13.998215826829304,
      "learning_rate": 3.839803107270509e-07,
      "logits/chosen": -0.8844782710075378,
      "logits/rejected": -0.7867937088012695,
      "logps/chosen": -551.641845703125,
      "logps/rejected": -744.5933227539062,
      "loss": 0.4074,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.376842737197876,
      "rewards/margins": 1.7052807807922363,
      "rewards/rejected": -5.082123756408691,
      "step": 244
    },
    {
      "epoch": 0.5236441357200107,
      "grad_norm": 14.905532531142882,
      "learning_rate": 3.813737581162017e-07,
      "logits/chosen": -1.0912367105484009,
      "logits/rejected": -1.0676392316818237,
      "logps/chosen": -624.3543701171875,
      "logps/rejected": -729.902099609375,
      "loss": 0.413,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5566658973693848,
      "rewards/margins": 1.0893948078155518,
      "rewards/rejected": -4.646060943603516,
      "step": 245
    },
    {
      "epoch": 0.5257814587229495,
      "grad_norm": 13.609157264860004,
      "learning_rate": 3.7876545014743065e-07,
      "logits/chosen": -0.898338258266449,
      "logits/rejected": -0.9181092381477356,
      "logps/chosen": -624.335205078125,
      "logps/rejected": -792.426025390625,
      "loss": 0.4068,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.840670108795166,
      "rewards/margins": 1.4304872751235962,
      "rewards/rejected": -5.2711567878723145,
      "step": 246
    },
    {
      "epoch": 0.5279187817258884,
      "grad_norm": 15.394561748839626,
      "learning_rate": 3.761555327552485e-07,
      "logits/chosen": -1.0056723356246948,
      "logits/rejected": -1.0341317653656006,
      "logps/chosen": -591.2755737304688,
      "logps/rejected": -716.7362060546875,
      "loss": 0.3879,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.308074951171875,
      "rewards/margins": 1.277435302734375,
      "rewards/rejected": -4.585509777069092,
      "step": 247
    },
    {
      "epoch": 0.5300561047288271,
      "grad_norm": 16.757770907784614,
      "learning_rate": 3.735441519642139e-07,
      "logits/chosen": -1.1397889852523804,
      "logits/rejected": -1.1101233959197998,
      "logps/chosen": -517.30810546875,
      "logps/rejected": -605.835205078125,
      "loss": 0.4748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.128363847732544,
      "rewards/margins": 0.9252746105194092,
      "rewards/rejected": -4.053638935089111,
      "step": 248
    },
    {
      "epoch": 0.5321934277317659,
      "grad_norm": 14.363837462668146,
      "learning_rate": 3.70931453880762e-07,
      "logits/chosen": -0.9677891135215759,
      "logits/rejected": -0.9882450699806213,
      "logps/chosen": -697.7656860351562,
      "logps/rejected": -846.583740234375,
      "loss": 0.4142,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.01542329788208,
      "rewards/margins": 1.4604833126068115,
      "rewards/rejected": -5.4759063720703125,
      "step": 249
    },
    {
      "epoch": 0.5343307507347048,
      "grad_norm": 18.905238772507477,
      "learning_rate": 3.6831758468503035e-07,
      "logits/chosen": -1.1917240619659424,
      "logits/rejected": -1.1658238172531128,
      "logps/chosen": -629.4384155273438,
      "logps/rejected": -744.068603515625,
      "loss": 0.5198,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.910032272338867,
      "rewards/margins": 1.0619316101074219,
      "rewards/rejected": -4.971964359283447,
      "step": 250
    },
    {
      "epoch": 0.5364680737376436,
      "grad_norm": 13.775541697952745,
      "learning_rate": 3.6570269062268025e-07,
      "logits/chosen": -0.8680559992790222,
      "logits/rejected": -0.8241602182388306,
      "logps/chosen": -813.6458740234375,
      "logps/rejected": -992.6861572265625,
      "loss": 0.3533,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.124260425567627,
      "rewards/margins": 1.764125108718872,
      "rewards/rejected": -6.88838529586792,
      "step": 251
    },
    {
      "epoch": 0.5386053967405824,
      "grad_norm": 22.37517195362321,
      "learning_rate": 3.6308691799671404e-07,
      "logits/chosen": -0.7893460392951965,
      "logits/rejected": -0.8846604824066162,
      "logps/chosen": -585.7125854492188,
      "logps/rejected": -700.5401611328125,
      "loss": 0.4298,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.845160961151123,
      "rewards/margins": 1.1050822734832764,
      "rewards/rejected": -4.9502434730529785,
      "step": 252
    },
    {
      "epoch": 0.5407427197435213,
      "grad_norm": 20.261154485800315,
      "learning_rate": 3.604704131592895e-07,
      "logits/chosen": -1.1260894536972046,
      "logits/rejected": -1.116862177848816,
      "logps/chosen": -766.9825439453125,
      "logps/rejected": -869.4150390625,
      "loss": 0.4666,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.645950794219971,
      "rewards/margins": 1.14671790599823,
      "rewards/rejected": -5.79266881942749,
      "step": 253
    },
    {
      "epoch": 0.5428800427464601,
      "grad_norm": 24.637498347289263,
      "learning_rate": 3.578533225035318e-07,
      "logits/chosen": -1.0664944648742676,
      "logits/rejected": -1.0649199485778809,
      "logps/chosen": -772.8311157226562,
      "logps/rejected": -995.858642578125,
      "loss": 0.3884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.625110149383545,
      "rewards/margins": 2.349405527114868,
      "rewards/rejected": -6.974515914916992,
      "step": 254
    },
    {
      "epoch": 0.5450173657493989,
      "grad_norm": 18.22340364830364,
      "learning_rate": 3.552357924553423e-07,
      "logits/chosen": -0.9930300712585449,
      "logits/rejected": -1.0258715152740479,
      "logps/chosen": -745.672119140625,
      "logps/rejected": -877.061279296875,
      "loss": 0.4139,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.158998489379883,
      "rewards/margins": 1.6085114479064941,
      "rewards/rejected": -5.767509937286377,
      "step": 255
    },
    {
      "epoch": 0.5471546887523377,
      "grad_norm": 16.289391048607587,
      "learning_rate": 3.526179694652065e-07,
      "logits/chosen": -1.0990363359451294,
      "logits/rejected": -1.0807216167449951,
      "logps/chosen": -646.4122314453125,
      "logps/rejected": -707.5439453125,
      "loss": 0.4572,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.093816757202148,
      "rewards/margins": 0.4628375768661499,
      "rewards/rejected": -4.556654453277588,
      "step": 256
    },
    {
      "epoch": 0.5492920117552765,
      "grad_norm": 20.631321302576914,
      "learning_rate": 3.5e-07,
      "logits/chosen": -1.1180474758148193,
      "logits/rejected": -1.0840083360671997,
      "logps/chosen": -714.1011352539062,
      "logps/rejected": -901.5518798828125,
      "loss": 0.5129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.527349948883057,
      "rewards/margins": 1.6243555545806885,
      "rewards/rejected": -6.151705265045166,
      "step": 257
    },
    {
      "epoch": 0.5514293347582153,
      "grad_norm": 17.30980385185334,
      "learning_rate": 3.4738203053479345e-07,
      "logits/chosen": -1.1147381067276,
      "logits/rejected": -1.0951800346374512,
      "logps/chosen": -671.677978515625,
      "logps/rejected": -869.009033203125,
      "loss": 0.3768,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.159421443939209,
      "rewards/margins": 1.6203829050064087,
      "rewards/rejected": -5.77980375289917,
      "step": 258
    },
    {
      "epoch": 0.5535666577611541,
      "grad_norm": 16.77670843921571,
      "learning_rate": 3.447642075446578e-07,
      "logits/chosen": -1.027138113975525,
      "logits/rejected": -1.0414177179336548,
      "logps/chosen": -600.3823852539062,
      "logps/rejected": -688.4371337890625,
      "loss": 0.3932,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.747048854827881,
      "rewards/margins": 0.837365448474884,
      "rewards/rejected": -4.584414482116699,
      "step": 259
    },
    {
      "epoch": 0.555703980764093,
      "grad_norm": 13.731293304469197,
      "learning_rate": 3.421466774964683e-07,
      "logits/chosen": -1.060163140296936,
      "logits/rejected": -1.1123335361480713,
      "logps/chosen": -769.50244140625,
      "logps/rejected": -878.5049438476562,
      "loss": 0.4123,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.471192359924316,
      "rewards/margins": 1.6601918935775757,
      "rewards/rejected": -6.131383895874023,
      "step": 260
    },
    {
      "epoch": 0.5578413037670318,
      "grad_norm": 21.94893337829568,
      "learning_rate": 3.395295868407106e-07,
      "logits/chosen": -0.7995742559432983,
      "logits/rejected": -0.7949234247207642,
      "logps/chosen": -540.7459716796875,
      "logps/rejected": -606.6634521484375,
      "loss": 0.4368,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -3.475754976272583,
      "rewards/margins": 0.5799713134765625,
      "rewards/rejected": -4.055726051330566,
      "step": 261
    },
    {
      "epoch": 0.5599786267699706,
      "grad_norm": 18.73889178987264,
      "learning_rate": 3.3691308200328604e-07,
      "logits/chosen": -0.8902988433837891,
      "logits/rejected": -0.9296258687973022,
      "logps/chosen": -738.5679931640625,
      "logps/rejected": -910.6119384765625,
      "loss": 0.4267,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.609591007232666,
      "rewards/margins": 1.8606202602386475,
      "rewards/rejected": -6.470211029052734,
      "step": 262
    },
    {
      "epoch": 0.5621159497729095,
      "grad_norm": 17.495724524128903,
      "learning_rate": 3.342973093773199e-07,
      "logits/chosen": -1.0284425020217896,
      "logits/rejected": -1.0994598865509033,
      "logps/chosen": -763.5560302734375,
      "logps/rejected": -916.3192749023438,
      "loss": 0.4193,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.318557262420654,
      "rewards/margins": 1.6308737993240356,
      "rewards/rejected": -5.949431419372559,
      "step": 263
    },
    {
      "epoch": 0.5642532727758482,
      "grad_norm": 21.8003372321031,
      "learning_rate": 3.316824153149696e-07,
      "logits/chosen": -0.9204785823822021,
      "logits/rejected": -0.9681702852249146,
      "logps/chosen": -642.7691650390625,
      "logps/rejected": -805.9739990234375,
      "loss": 0.4396,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.044567108154297,
      "rewards/margins": 1.6253376007080078,
      "rewards/rejected": -5.669903755187988,
      "step": 264
    },
    {
      "epoch": 0.566390595778787,
      "grad_norm": 29.221196391296576,
      "learning_rate": 3.29068546119238e-07,
      "logits/chosen": -0.8801388740539551,
      "logits/rejected": -0.8704509139060974,
      "logps/chosen": -707.189453125,
      "logps/rejected": -804.7384033203125,
      "loss": 0.597,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -4.639124870300293,
      "rewards/margins": 1.1081290245056152,
      "rewards/rejected": -5.74725341796875,
      "step": 265
    },
    {
      "epoch": 0.5685279187817259,
      "grad_norm": 21.440238859390956,
      "learning_rate": 3.2645584803578614e-07,
      "logits/chosen": -0.9265174865722656,
      "logits/rejected": -0.9847102761268616,
      "logps/chosen": -483.4872131347656,
      "logps/rejected": -527.4281005859375,
      "loss": 0.44,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.243124008178711,
      "rewards/margins": 0.43065446615219116,
      "rewards/rejected": -3.6737782955169678,
      "step": 266
    },
    {
      "epoch": 0.5706652417846647,
      "grad_norm": 15.539782944871728,
      "learning_rate": 3.238444672447515e-07,
      "logits/chosen": -0.8670752048492432,
      "logits/rejected": -0.8761327862739563,
      "logps/chosen": -669.8744506835938,
      "logps/rejected": -792.420166015625,
      "loss": 0.4026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.5057759284973145,
      "rewards/margins": 1.2092504501342773,
      "rewards/rejected": -5.715025901794434,
      "step": 267
    },
    {
      "epoch": 0.5728025647876035,
      "grad_norm": 15.529372167608297,
      "learning_rate": 3.2123454985256943e-07,
      "logits/chosen": -0.9794555902481079,
      "logits/rejected": -0.9547824859619141,
      "logps/chosen": -610.2971801757812,
      "logps/rejected": -825.6856079101562,
      "loss": 0.3279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7575111389160156,
      "rewards/margins": 2.0598204135894775,
      "rewards/rejected": -5.8173322677612305,
      "step": 268
    },
    {
      "epoch": 0.5749398877905424,
      "grad_norm": 22.72130516871319,
      "learning_rate": 3.186262418837983e-07,
      "logits/chosen": -1.082902431488037,
      "logits/rejected": -1.1030265092849731,
      "logps/chosen": -545.2824096679688,
      "logps/rejected": -609.6895141601562,
      "loss": 0.5641,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.404395580291748,
      "rewards/margins": 0.5698573589324951,
      "rewards/rejected": -3.974252939224243,
      "step": 269
    },
    {
      "epoch": 0.5770772107934812,
      "grad_norm": 16.252579751792183,
      "learning_rate": 3.1601968927294914e-07,
      "logits/chosen": -0.9424474239349365,
      "logits/rejected": -0.9634624123573303,
      "logps/chosen": -599.9176025390625,
      "logps/rejected": -702.5125732421875,
      "loss": 0.4418,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.833646774291992,
      "rewards/margins": 0.9874881505966187,
      "rewards/rejected": -4.821134567260742,
      "step": 270
    },
    {
      "epoch": 0.57921453379642,
      "grad_norm": 15.206487367829114,
      "learning_rate": 3.134150378563213e-07,
      "logits/chosen": -0.9836793541908264,
      "logits/rejected": -0.9680826663970947,
      "logps/chosen": -637.9475708007812,
      "logps/rejected": -767.5353393554688,
      "loss": 0.3601,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.120051383972168,
      "rewards/margins": 1.235574722290039,
      "rewards/rejected": -5.355626106262207,
      "step": 271
    },
    {
      "epoch": 0.5813518567993589,
      "grad_norm": 21.078417332237123,
      "learning_rate": 3.1081243336384226e-07,
      "logits/chosen": -0.9045091271400452,
      "logits/rejected": -0.9048910140991211,
      "logps/chosen": -615.5716552734375,
      "logps/rejected": -752.4681396484375,
      "loss": 0.3887,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.629241704940796,
      "rewards/margins": 1.4587962627410889,
      "rewards/rejected": -5.088037490844727,
      "step": 272
    },
    {
      "epoch": 0.5834891798022976,
      "grad_norm": 17.81077343546014,
      "learning_rate": 3.0821202141091453e-07,
      "logits/chosen": -1.1009454727172852,
      "logits/rejected": -0.9980039596557617,
      "logps/chosen": -591.7613525390625,
      "logps/rejected": -779.107421875,
      "loss": 0.4669,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.549128770828247,
      "rewards/margins": 1.750599980354309,
      "rewards/rejected": -5.299729824066162,
      "step": 273
    },
    {
      "epoch": 0.5856265028052364,
      "grad_norm": 19.256523288494076,
      "learning_rate": 3.0561394749026814e-07,
      "logits/chosen": -1.008500576019287,
      "logits/rejected": -0.9249081611633301,
      "logps/chosen": -631.9851684570312,
      "logps/rejected": -834.7702026367188,
      "loss": 0.4028,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.780165195465088,
      "rewards/margins": 1.7658891677856445,
      "rewards/rejected": -5.546054363250732,
      "step": 274
    },
    {
      "epoch": 0.5877638258081752,
      "grad_norm": 21.051631953087565,
      "learning_rate": 3.030183569638207e-07,
      "logits/chosen": -1.1362390518188477,
      "logits/rejected": -1.0792036056518555,
      "logps/chosen": -576.5594482421875,
      "logps/rejected": -694.7816162109375,
      "loss": 0.4611,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.658750534057617,
      "rewards/margins": 1.0495951175689697,
      "rewards/rejected": -4.708345890045166,
      "step": 275
    },
    {
      "epoch": 0.5899011488111141,
      "grad_norm": 15.76447884471548,
      "learning_rate": 3.0042539505454386e-07,
      "logits/chosen": -0.9091801643371582,
      "logits/rejected": -0.9027244448661804,
      "logps/chosen": -592.4364013671875,
      "logps/rejected": -717.2314453125,
      "loss": 0.3772,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.7294676303863525,
      "rewards/margins": 1.2672703266143799,
      "rewards/rejected": -4.996737957000732,
      "step": 276
    },
    {
      "epoch": 0.5920384718140529,
      "grad_norm": 14.607600516553031,
      "learning_rate": 2.978352068383389e-07,
      "logits/chosen": -0.9434270858764648,
      "logits/rejected": -0.9658694267272949,
      "logps/chosen": -426.6009216308594,
      "logps/rejected": -538.635009765625,
      "loss": 0.3716,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.5574347972869873,
      "rewards/margins": 1.0464235544204712,
      "rewards/rejected": -3.603858470916748,
      "step": 277
    },
    {
      "epoch": 0.5941757948169917,
      "grad_norm": 20.680956313480134,
      "learning_rate": 2.952479372359191e-07,
      "logits/chosen": -0.9002370238304138,
      "logits/rejected": -0.9020406007766724,
      "logps/chosen": -564.5108642578125,
      "logps/rejected": -660.9511108398438,
      "loss": 0.3884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.561006784439087,
      "rewards/margins": 0.8000065088272095,
      "rewards/rejected": -4.361013412475586,
      "step": 278
    },
    {
      "epoch": 0.5963131178199306,
      "grad_norm": 27.6993585691743,
      "learning_rate": 2.926637310047018e-07,
      "logits/chosen": -1.1155834197998047,
      "logits/rejected": -1.105792760848999,
      "logps/chosen": -596.0205078125,
      "logps/rejected": -664.81640625,
      "loss": 0.3998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.8352932929992676,
      "rewards/margins": 0.8193560838699341,
      "rewards/rejected": -4.654649257659912,
      "step": 279
    },
    {
      "epoch": 0.5984504408228694,
      "grad_norm": 18.916329978828376,
      "learning_rate": 2.900827327307087e-07,
      "logits/chosen": -1.2077603340148926,
      "logits/rejected": -1.0835894346237183,
      "logps/chosen": -778.5531005859375,
      "logps/rejected": -890.9183349609375,
      "loss": 0.4658,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.6639227867126465,
      "rewards/margins": 1.0659689903259277,
      "rewards/rejected": -5.729891777038574,
      "step": 280
    },
    {
      "epoch": 0.6005877638258081,
      "grad_norm": 18.365489297240394,
      "learning_rate": 2.8750508682047717e-07,
      "logits/chosen": -1.1601464748382568,
      "logits/rejected": -1.1277116537094116,
      "logps/chosen": -677.4540405273438,
      "logps/rejected": -838.7645874023438,
      "loss": 0.4918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.8409969806671143,
      "rewards/margins": 1.480958342552185,
      "rewards/rejected": -5.321955680847168,
      "step": 281
    },
    {
      "epoch": 0.602725086828747,
      "grad_norm": 20.141526963645916,
      "learning_rate": 2.849309374929799e-07,
      "logits/chosen": -1.1542294025421143,
      "logits/rejected": -1.1290372610092163,
      "logps/chosen": -638.0623779296875,
      "logps/rejected": -759.2158203125,
      "loss": 0.4268,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.9765753746032715,
      "rewards/margins": 1.2076830863952637,
      "rewards/rejected": -5.184258460998535,
      "step": 282
    },
    {
      "epoch": 0.6048624098316858,
      "grad_norm": 17.776205208549275,
      "learning_rate": 2.823604287715564e-07,
      "logits/chosen": -0.9990053772926331,
      "logits/rejected": -1.038635015487671,
      "logps/chosen": -641.4173583984375,
      "logps/rejected": -752.838623046875,
      "loss": 0.3768,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.788572072982788,
      "rewards/margins": 1.1508256196975708,
      "rewards/rejected": -4.939397811889648,
      "step": 283
    },
    {
      "epoch": 0.6069997328346246,
      "grad_norm": 26.51717001409862,
      "learning_rate": 2.797937044758545e-07,
      "logits/chosen": -0.9398884177207947,
      "logits/rejected": -0.9421570897102356,
      "logps/chosen": -616.0810546875,
      "logps/rejected": -697.7490844726562,
      "loss": 0.4983,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -4.009638786315918,
      "rewards/margins": 0.752707302570343,
      "rewards/rejected": -4.762345790863037,
      "step": 284
    },
    {
      "epoch": 0.6091370558375635,
      "grad_norm": 35.30303590344862,
      "learning_rate": 2.772309082137843e-07,
      "logits/chosen": -1.085971474647522,
      "logits/rejected": -1.0797044038772583,
      "logps/chosen": -635.1014404296875,
      "logps/rejected": -687.0366821289062,
      "loss": 0.5866,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.941286325454712,
      "rewards/margins": 0.7452540397644043,
      "rewards/rejected": -4.686540603637695,
      "step": 285
    },
    {
      "epoch": 0.6112743788405023,
      "grad_norm": 26.601600287549388,
      "learning_rate": 2.7467218337348233e-07,
      "logits/chosen": -0.8175097703933716,
      "logits/rejected": -0.7498407959938049,
      "logps/chosen": -589.5838623046875,
      "logps/rejected": -743.729248046875,
      "loss": 0.4928,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.852874279022217,
      "rewards/margins": 1.4163994789123535,
      "rewards/rejected": -5.2692742347717285,
      "step": 286
    },
    {
      "epoch": 0.6134117018434411,
      "grad_norm": 17.687526214496195,
      "learning_rate": 2.7211767311529e-07,
      "logits/chosen": -0.8146362900733948,
      "logits/rejected": -0.8423186540603638,
      "logps/chosen": -607.890869140625,
      "logps/rejected": -708.3568115234375,
      "loss": 0.4853,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.738939046859741,
      "rewards/margins": 0.9543510675430298,
      "rewards/rejected": -4.693289756774902,
      "step": 287
    },
    {
      "epoch": 0.6155490248463799,
      "grad_norm": 18.930801664854705,
      "learning_rate": 2.695675203637432e-07,
      "logits/chosen": -1.0421204566955566,
      "logits/rejected": -1.0792558193206787,
      "logps/chosen": -602.5674438476562,
      "logps/rejected": -736.7977905273438,
      "loss": 0.4063,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.712022304534912,
      "rewards/margins": 1.2383270263671875,
      "rewards/rejected": -4.9503493309021,
      "step": 288
    },
    {
      "epoch": 0.6176863478493188,
      "grad_norm": 17.90358421564856,
      "learning_rate": 2.670218677995759e-07,
      "logits/chosen": -1.0456268787384033,
      "logits/rejected": -0.9645753502845764,
      "logps/chosen": -618.1510009765625,
      "logps/rejected": -819.269287109375,
      "loss": 0.4206,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5795562267303467,
      "rewards/margins": 2.006627321243286,
      "rewards/rejected": -5.586184024810791,
      "step": 289
    },
    {
      "epoch": 0.6198236708522575,
      "grad_norm": 23.380000713489068,
      "learning_rate": 2.6448085785173703e-07,
      "logits/chosen": -1.005885124206543,
      "logits/rejected": -0.964793860912323,
      "logps/chosen": -665.0631103515625,
      "logps/rejected": -754.4349975585938,
      "loss": 0.4141,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.058958053588867,
      "rewards/margins": 0.8748332262039185,
      "rewards/rejected": -4.933791160583496,
      "step": 290
    },
    {
      "epoch": 0.6219609938551963,
      "grad_norm": 16.59292554666212,
      "learning_rate": 2.6194463268942203e-07,
      "logits/chosen": -1.0780235528945923,
      "logits/rejected": -1.097104787826538,
      "logps/chosen": -590.5000610351562,
      "logps/rejected": -796.20166015625,
      "loss": 0.3928,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4696645736694336,
      "rewards/margins": 1.882230281829834,
      "rewards/rejected": -5.351894855499268,
      "step": 291
    },
    {
      "epoch": 0.6240983168581352,
      "grad_norm": 14.429424103488177,
      "learning_rate": 2.594133342141177e-07,
      "logits/chosen": -0.9270427227020264,
      "logits/rejected": -0.8782027959823608,
      "logps/chosen": -498.6771545410156,
      "logps/rejected": -686.7008056640625,
      "loss": 0.3666,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.730340003967285,
      "rewards/margins": 1.7681416273117065,
      "rewards/rejected": -4.498481273651123,
      "step": 292
    },
    {
      "epoch": 0.626235639861074,
      "grad_norm": 21.064661843353115,
      "learning_rate": 2.568871040516637e-07,
      "logits/chosen": -1.0022443532943726,
      "logits/rejected": -0.9993023872375488,
      "logps/chosen": -685.757080078125,
      "logps/rejected": -779.1624755859375,
      "loss": 0.4839,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.1098246574401855,
      "rewards/margins": 0.944150984287262,
      "rewards/rejected": -5.053975582122803,
      "step": 293
    },
    {
      "epoch": 0.6283729628640128,
      "grad_norm": 19.087071179960915,
      "learning_rate": 2.54366083544328e-07,
      "logits/chosen": -0.8884956240653992,
      "logits/rejected": -0.9064831137657166,
      "logps/chosen": -571.4553833007812,
      "logps/rejected": -674.0166625976562,
      "loss": 0.504,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.6071152687072754,
      "rewards/margins": 0.9564936757087708,
      "rewards/rejected": -4.563608646392822,
      "step": 294
    },
    {
      "epoch": 0.6305102858669517,
      "grad_norm": 21.794412344839774,
      "learning_rate": 2.5185041374289907e-07,
      "logits/chosen": -1.1455574035644531,
      "logits/rejected": -1.1415196657180786,
      "logps/chosen": -497.8589782714844,
      "logps/rejected": -751.036376953125,
      "loss": 0.4041,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.744825839996338,
      "rewards/margins": 2.1894543170928955,
      "rewards/rejected": -4.934279918670654,
      "step": 295
    },
    {
      "epoch": 0.6326476088698905,
      "grad_norm": 19.554010929778244,
      "learning_rate": 2.4934023539879395e-07,
      "logits/chosen": -0.979045569896698,
      "logits/rejected": -0.9661969542503357,
      "logps/chosen": -601.6738891601562,
      "logps/rejected": -756.327392578125,
      "loss": 0.3696,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.3881332874298096,
      "rewards/margins": 1.3837690353393555,
      "rewards/rejected": -4.771902561187744,
      "step": 296
    },
    {
      "epoch": 0.6347849318728293,
      "grad_norm": 20.678635374709422,
      "learning_rate": 2.468356889561835e-07,
      "logits/chosen": -1.0623462200164795,
      "logits/rejected": -1.0434157848358154,
      "logps/chosen": -569.0560302734375,
      "logps/rejected": -698.5628051757812,
      "loss": 0.3802,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.12467622756958,
      "rewards/margins": 1.3073335886001587,
      "rewards/rejected": -4.432010173797607,
      "step": 297
    },
    {
      "epoch": 0.6369222548757681,
      "grad_norm": 15.887777705629667,
      "learning_rate": 2.443369145441343e-07,
      "logits/chosen": -1.0596822500228882,
      "logits/rejected": -1.044713020324707,
      "logps/chosen": -531.6007080078125,
      "logps/rejected": -636.859130859375,
      "loss": 0.3491,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.2483232021331787,
      "rewards/margins": 1.0054796934127808,
      "rewards/rejected": -4.253802299499512,
      "step": 298
    },
    {
      "epoch": 0.6390595778787069,
      "grad_norm": 20.882809623209052,
      "learning_rate": 2.418440519687684e-07,
      "logits/chosen": -1.0757739543914795,
      "logits/rejected": -1.0567525625228882,
      "logps/chosen": -511.7173156738281,
      "logps/rejected": -668.195556640625,
      "loss": 0.4593,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.052992820739746,
      "rewards/margins": 1.4922919273376465,
      "rewards/rejected": -4.545284271240234,
      "step": 299
    },
    {
      "epoch": 0.6411969008816457,
      "grad_norm": 14.618996262053995,
      "learning_rate": 2.393572407054416e-07,
      "logits/chosen": -0.8958306312561035,
      "logits/rejected": -0.8867323398590088,
      "logps/chosen": -659.5025634765625,
      "logps/rejected": -834.076171875,
      "loss": 0.3802,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.8700366020202637,
      "rewards/margins": 1.7134443521499634,
      "rewards/rejected": -5.5834808349609375,
      "step": 300
    },
    {
      "epoch": 0.6411969008816457,
      "eval_logits/chosen": -1.1104539632797241,
      "eval_logits/rejected": -1.130950927734375,
      "eval_logps/chosen": -652.87158203125,
      "eval_logps/rejected": -784.0881958007812,
      "eval_loss": 0.40694576501846313,
      "eval_rewards/accuracies": 0.8427419066429138,
      "eval_rewards/chosen": -3.7974159717559814,
      "eval_rewards/margins": 1.3731279373168945,
      "eval_rewards/rejected": -5.170544147491455,
      "eval_runtime": 127.0172,
      "eval_samples_per_second": 15.439,
      "eval_steps_per_second": 0.488,
      "step": 300
    },
    {
      "epoch": 0.6433342238845846,
      "grad_norm": 19.43800281730569,
      "learning_rate": 2.3687661989093904e-07,
      "logits/chosen": -0.9887138605117798,
      "logits/rejected": -1.0439525842666626,
      "logps/chosen": -595.8880615234375,
      "logps/rejected": -732.6058349609375,
      "loss": 0.4198,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.636826753616333,
      "rewards/margins": 1.4018120765686035,
      "rewards/rejected": -5.038639068603516,
      "step": 301
    },
    {
      "epoch": 0.6454715468875234,
      "grad_norm": 22.456716854896456,
      "learning_rate": 2.3440232831569162e-07,
      "logits/chosen": -1.0877864360809326,
      "logits/rejected": -1.1251873970031738,
      "logps/chosen": -665.487548828125,
      "logps/rejected": -819.5171508789062,
      "loss": 0.4536,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.9592628479003906,
      "rewards/margins": 1.4847943782806396,
      "rewards/rejected": -5.444056987762451,
      "step": 302
    },
    {
      "epoch": 0.6476088698904622,
      "grad_norm": 16.0762099994994,
      "learning_rate": 2.319345044160095e-07,
      "logits/chosen": -1.2076233625411987,
      "logits/rejected": -1.230367660522461,
      "logps/chosen": -769.3408203125,
      "logps/rejected": -1005.293701171875,
      "loss": 0.4059,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -4.611854076385498,
      "rewards/margins": 2.47025465965271,
      "rewards/rejected": -7.082108974456787,
      "step": 303
    },
    {
      "epoch": 0.649746192893401,
      "grad_norm": 17.96340906701677,
      "learning_rate": 2.2947328626633766e-07,
      "logits/chosen": -1.058774709701538,
      "logits/rejected": -1.0470080375671387,
      "logps/chosen": -590.3370361328125,
      "logps/rejected": -681.3898315429688,
      "loss": 0.4649,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.8478586673736572,
      "rewards/margins": 0.7897017002105713,
      "rewards/rejected": -4.637560844421387,
      "step": 304
    },
    {
      "epoch": 0.6518835158963399,
      "grad_norm": 22.719738681967296,
      "learning_rate": 2.2701881157153e-07,
      "logits/chosen": -1.1047711372375488,
      "logits/rejected": -1.0974376201629639,
      "logps/chosen": -653.9512329101562,
      "logps/rejected": -839.6368408203125,
      "loss": 0.3939,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.9316320419311523,
      "rewards/margins": 1.9634764194488525,
      "rewards/rejected": -5.895109176635742,
      "step": 305
    },
    {
      "epoch": 0.6540208388992786,
      "grad_norm": 17.780668266782456,
      "learning_rate": 2.245712176591449e-07,
      "logits/chosen": -1.0736472606658936,
      "logits/rejected": -1.0763237476348877,
      "logps/chosen": -615.7779541015625,
      "logps/rejected": -687.5728759765625,
      "loss": 0.4217,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.957977533340454,
      "rewards/margins": 0.773561954498291,
      "rewards/rejected": -4.731539726257324,
      "step": 306
    },
    {
      "epoch": 0.6561581619022174,
      "grad_norm": 23.74418488307736,
      "learning_rate": 2.2213064147176174e-07,
      "logits/chosen": -1.094491958618164,
      "logits/rejected": -1.1138453483581543,
      "logps/chosen": -725.0467529296875,
      "logps/rejected": -822.287353515625,
      "loss": 0.4067,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.600536823272705,
      "rewards/margins": 1.1523770093917847,
      "rewards/rejected": -5.752913951873779,
      "step": 307
    },
    {
      "epoch": 0.6582954849051563,
      "grad_norm": 15.5978905581422,
      "learning_rate": 2.196972195593192e-07,
      "logits/chosen": -1.1050280332565308,
      "logits/rejected": -1.0931296348571777,
      "logps/chosen": -582.18115234375,
      "logps/rejected": -741.2723388671875,
      "loss": 0.3451,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.750410318374634,
      "rewards/margins": 1.5357167720794678,
      "rewards/rejected": -5.28612756729126,
      "step": 308
    },
    {
      "epoch": 0.6604328079080951,
      "grad_norm": 17.049613660365612,
      "learning_rate": 2.1727108807147506e-07,
      "logits/chosen": -1.0725035667419434,
      "logits/rejected": -1.1041122674942017,
      "logps/chosen": -610.0225219726562,
      "logps/rejected": -762.8485107421875,
      "loss": 0.4524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.9435532093048096,
      "rewards/margins": 1.720339059829712,
      "rewards/rejected": -5.663891792297363,
      "step": 309
    },
    {
      "epoch": 0.6625701309110339,
      "grad_norm": 21.589679932544243,
      "learning_rate": 2.148523827499888e-07,
      "logits/chosen": -0.9776382446289062,
      "logits/rejected": -1.0473860502243042,
      "logps/chosen": -845.308837890625,
      "logps/rejected": -1063.19482421875,
      "loss": 0.4161,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.683403491973877,
      "rewards/margins": 1.968895673751831,
      "rewards/rejected": -7.652299404144287,
      "step": 310
    },
    {
      "epoch": 0.6647074539139728,
      "grad_norm": 20.546655366561865,
      "learning_rate": 2.1244123892112674e-07,
      "logits/chosen": -0.971989095211029,
      "logits/rejected": -0.9635407328605652,
      "logps/chosen": -752.0794677734375,
      "logps/rejected": -896.9614868164062,
      "loss": 0.4412,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.961668968200684,
      "rewards/margins": 1.418576717376709,
      "rewards/rejected": -6.380245685577393,
      "step": 311
    },
    {
      "epoch": 0.6668447769169116,
      "grad_norm": 16.64812094871957,
      "learning_rate": 2.1003779148809072e-07,
      "logits/chosen": -0.9267586469650269,
      "logits/rejected": -0.9304214715957642,
      "logps/chosen": -742.295166015625,
      "logps/rejected": -886.9481201171875,
      "loss": 0.3719,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.672271251678467,
      "rewards/margins": 1.6211450099945068,
      "rewards/rejected": -6.2934160232543945,
      "step": 312
    },
    {
      "epoch": 0.6689820999198504,
      "grad_norm": 21.010619081649534,
      "learning_rate": 2.0764217492346996e-07,
      "logits/chosen": -1.0951707363128662,
      "logits/rejected": -1.0666468143463135,
      "logps/chosen": -790.2965087890625,
      "logps/rejected": -910.8655395507812,
      "loss": 0.4601,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -5.142488956451416,
      "rewards/margins": 1.2062201499938965,
      "rewards/rejected": -6.348708629608154,
      "step": 313
    },
    {
      "epoch": 0.6711194229227893,
      "grad_norm": 19.84084802689762,
      "learning_rate": 2.0525452326171784e-07,
      "logits/chosen": -1.0453927516937256,
      "logits/rejected": -1.0439743995666504,
      "logps/chosen": -865.6875610351562,
      "logps/rejected": -1150.561767578125,
      "loss": 0.4036,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.613504409790039,
      "rewards/margins": 2.756046772003174,
      "rewards/rejected": -8.369551658630371,
      "step": 314
    },
    {
      "epoch": 0.673256745925728,
      "grad_norm": 17.469930397973854,
      "learning_rate": 2.028749700916521e-07,
      "logits/chosen": -0.955051600933075,
      "logits/rejected": -0.944666862487793,
      "logps/chosen": -787.5214233398438,
      "logps/rejected": -877.498779296875,
      "loss": 0.403,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -5.070991039276123,
      "rewards/margins": 0.8326537013053894,
      "rewards/rejected": -5.903645038604736,
      "step": 315
    },
    {
      "epoch": 0.6753940689286668,
      "grad_norm": 31.931815437246083,
      "learning_rate": 2.0050364854898127e-07,
      "logits/chosen": -1.066016674041748,
      "logits/rejected": -1.063683032989502,
      "logps/chosen": -740.451416015625,
      "logps/rejected": -935.1329345703125,
      "loss": 0.497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.710479736328125,
      "rewards/margins": 1.9114099740982056,
      "rewards/rejected": -6.621890068054199,
      "step": 316
    },
    {
      "epoch": 0.6775313919316056,
      "grad_norm": 23.195446430699608,
      "learning_rate": 1.9814069130885467e-07,
      "logits/chosen": -0.7954821586608887,
      "logits/rejected": -0.8669359683990479,
      "logps/chosen": -679.1461791992188,
      "logps/rejected": -868.0947875976562,
      "loss": 0.4159,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.395887851715088,
      "rewards/margins": 2.0239951610565186,
      "rewards/rejected": -6.4198832511901855,
      "step": 317
    },
    {
      "epoch": 0.6796687149345445,
      "grad_norm": 27.100165283554862,
      "learning_rate": 1.9578623057844064e-07,
      "logits/chosen": -1.1223851442337036,
      "logits/rejected": -1.1304949522018433,
      "logps/chosen": -606.3875732421875,
      "logps/rejected": -768.2408447265625,
      "loss": 0.4085,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.690753936767578,
      "rewards/margins": 1.4597177505493164,
      "rewards/rejected": -5.1504716873168945,
      "step": 318
    },
    {
      "epoch": 0.6818060379374833,
      "grad_norm": 19.59121701664144,
      "learning_rate": 1.9344039808952853e-07,
      "logits/chosen": -0.9872533082962036,
      "logits/rejected": -0.934008002281189,
      "logps/chosen": -740.6689453125,
      "logps/rejected": -1047.7109375,
      "loss": 0.3775,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.856841087341309,
      "rewards/margins": 2.8627772331237793,
      "rewards/rejected": -7.71961784362793,
      "step": 319
    },
    {
      "epoch": 0.6839433609404221,
      "grad_norm": 20.56349471919662,
      "learning_rate": 1.9110332509115864e-07,
      "logits/chosen": -1.0536935329437256,
      "logits/rejected": -1.111939549446106,
      "logps/chosen": -713.4016723632812,
      "logps/rejected": -897.5386962890625,
      "loss": 0.4077,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.394659996032715,
      "rewards/margins": 1.7375450134277344,
      "rewards/rejected": -6.132205009460449,
      "step": 320
    },
    {
      "epoch": 0.686080683943361,
      "grad_norm": 21.59919721276828,
      "learning_rate": 1.8877514234227874e-07,
      "logits/chosen": -0.8561320900917053,
      "logits/rejected": -0.9341483116149902,
      "logps/chosen": -669.7821044921875,
      "logps/rejected": -811.287353515625,
      "loss": 0.4224,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.155871391296387,
      "rewards/margins": 1.4691696166992188,
      "rewards/rejected": -5.6250410079956055,
      "step": 321
    },
    {
      "epoch": 0.6882180069462998,
      "grad_norm": 19.329148607505996,
      "learning_rate": 1.8645598010442828e-07,
      "logits/chosen": -0.9411246180534363,
      "logits/rejected": -0.9587753415107727,
      "logps/chosen": -699.0866088867188,
      "logps/rejected": -864.09375,
      "loss": 0.3697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.35011100769043,
      "rewards/margins": 1.8306729793548584,
      "rewards/rejected": -6.180783748626709,
      "step": 322
    },
    {
      "epoch": 0.6903553299492385,
      "grad_norm": 16.796887314919882,
      "learning_rate": 1.8414596813445047e-07,
      "logits/chosen": -0.9905204772949219,
      "logits/rejected": -1.024290680885315,
      "logps/chosen": -622.2347412109375,
      "logps/rejected": -708.5049438476562,
      "loss": 0.407,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.9594295024871826,
      "rewards/margins": 0.9213578104972839,
      "rewards/rejected": -4.8807878494262695,
      "step": 323
    },
    {
      "epoch": 0.6924926529521774,
      "grad_norm": 15.1418538158911,
      "learning_rate": 1.8184523567723194e-07,
      "logits/chosen": -1.0796232223510742,
      "logits/rejected": -1.0750352144241333,
      "logps/chosen": -628.67041015625,
      "logps/rejected": -812.5968017578125,
      "loss": 0.351,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.8600831031799316,
      "rewards/margins": 1.7709476947784424,
      "rewards/rejected": -5.631031036376953,
      "step": 324
    },
    {
      "epoch": 0.6946299759551162,
      "grad_norm": 18.026973060407244,
      "learning_rate": 1.7955391145847145e-07,
      "logits/chosen": -0.9602807760238647,
      "logits/rejected": -0.9538272619247437,
      "logps/chosen": -533.8419189453125,
      "logps/rejected": -605.8224487304688,
      "loss": 0.338,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.626598596572876,
      "rewards/margins": 0.6452375054359436,
      "rewards/rejected": -4.271836280822754,
      "step": 325
    },
    {
      "epoch": 0.696767298958055,
      "grad_norm": 20.22808705229649,
      "learning_rate": 1.772721236774786e-07,
      "logits/chosen": -0.8902896642684937,
      "logits/rejected": -0.891946017742157,
      "logps/chosen": -547.3910522460938,
      "logps/rejected": -737.225830078125,
      "loss": 0.3568,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.2871787548065186,
      "rewards/margins": 1.8806804418563843,
      "rewards/rejected": -5.167859077453613,
      "step": 326
    },
    {
      "epoch": 0.6989046219609939,
      "grad_norm": 24.61898214966437,
      "learning_rate": 1.7500000000000007e-07,
      "logits/chosen": -1.0208594799041748,
      "logits/rejected": -0.9971433877944946,
      "logps/chosen": -628.2722778320312,
      "logps/rejected": -766.7388916015625,
      "loss": 0.4498,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.7911126613616943,
      "rewards/margins": 1.395633339881897,
      "rewards/rejected": -5.186745643615723,
      "step": 327
    },
    {
      "epoch": 0.7010419449639327,
      "grad_norm": 23.202132151091234,
      "learning_rate": 1.7273766755107707e-07,
      "logits/chosen": -1.00026535987854,
      "logits/rejected": -1.0049934387207031,
      "logps/chosen": -618.7490234375,
      "logps/rejected": -784.1082153320312,
      "loss": 0.4044,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.9590413570404053,
      "rewards/margins": 1.4626474380493164,
      "rewards/rejected": -5.421689033508301,
      "step": 328
    },
    {
      "epoch": 0.7031792679668715,
      "grad_norm": 22.040873223414177,
      "learning_rate": 1.7048525290793284e-07,
      "logits/chosen": -1.151913046836853,
      "logits/rejected": -1.1811047792434692,
      "logps/chosen": -676.8132934570312,
      "logps/rejected": -835.91064453125,
      "loss": 0.4432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.948080062866211,
      "rewards/margins": 1.632805347442627,
      "rewards/rejected": -5.580885887145996,
      "step": 329
    },
    {
      "epoch": 0.7053165909698104,
      "grad_norm": 16.489066064805076,
      "learning_rate": 1.6824288209289077e-07,
      "logits/chosen": -0.9843920469284058,
      "logits/rejected": -0.9963368773460388,
      "logps/chosen": -695.6856689453125,
      "logps/rejected": -835.2234497070312,
      "loss": 0.3754,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.025155067443848,
      "rewards/margins": 1.4948614835739136,
      "rewards/rejected": -5.520016193389893,
      "step": 330
    },
    {
      "epoch": 0.7074539139727491,
      "grad_norm": 18.164643888570296,
      "learning_rate": 1.660106805663233e-07,
      "logits/chosen": -0.9996950626373291,
      "logits/rejected": -1.0081021785736084,
      "logps/chosen": -585.10009765625,
      "logps/rejected": -739.9697265625,
      "loss": 0.3477,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.4289603233337402,
      "rewards/margins": 1.5738253593444824,
      "rewards/rejected": -5.002785682678223,
      "step": 331
    },
    {
      "epoch": 0.7095912369756879,
      "grad_norm": 18.104299917352034,
      "learning_rate": 1.6378877321963224e-07,
      "logits/chosen": -1.0804736614227295,
      "logits/rejected": -1.0568981170654297,
      "logps/chosen": -713.994140625,
      "logps/rejected": -918.4098510742188,
      "loss": 0.3097,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.44266414642334,
      "rewards/margins": 1.9913928508758545,
      "rewards/rejected": -6.434057712554932,
      "step": 332
    },
    {
      "epoch": 0.7117285599786267,
      "grad_norm": 19.250777560445798,
      "learning_rate": 1.6157728436826125e-07,
      "logits/chosen": -1.0584830045700073,
      "logits/rejected": -1.0436077117919922,
      "logps/chosen": -645.3274536132812,
      "logps/rejected": -729.302001953125,
      "loss": 0.3813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.107142925262451,
      "rewards/margins": 0.8789306879043579,
      "rewards/rejected": -4.9860734939575195,
      "step": 333
    },
    {
      "epoch": 0.7138658829815656,
      "grad_norm": 18.363434689830413,
      "learning_rate": 1.5937633774474053e-07,
      "logits/chosen": -0.964330792427063,
      "logits/rejected": -1.0107676982879639,
      "logps/chosen": -681.0447387695312,
      "logps/rejected": -893.3312377929688,
      "loss": 0.4289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.495457172393799,
      "rewards/margins": 2.035775661468506,
      "rewards/rejected": -6.531231880187988,
      "step": 334
    },
    {
      "epoch": 0.7160032059845044,
      "grad_norm": 19.36554980671261,
      "learning_rate": 1.5718605649176415e-07,
      "logits/chosen": -1.08909273147583,
      "logits/rejected": -1.029083013534546,
      "logps/chosen": -606.0968017578125,
      "logps/rejected": -801.4658203125,
      "loss": 0.3958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.555162191390991,
      "rewards/margins": 1.8386735916137695,
      "rewards/rejected": -5.393835544586182,
      "step": 335
    },
    {
      "epoch": 0.7181405289874432,
      "grad_norm": 19.53200074466149,
      "learning_rate": 1.5500656315529971e-07,
      "logits/chosen": -0.9612129330635071,
      "logits/rejected": -1.0220246315002441,
      "logps/chosen": -689.0376586914062,
      "logps/rejected": -871.2042846679688,
      "loss": 0.4066,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.374889850616455,
      "rewards/margins": 1.9528312683105469,
      "rewards/rejected": -6.327720642089844,
      "step": 336
    },
    {
      "epoch": 0.7202778519903821,
      "grad_norm": 16.525872426498882,
      "learning_rate": 1.5283797967773228e-07,
      "logits/chosen": -0.9871097803115845,
      "logits/rejected": -1.0105241537094116,
      "logps/chosen": -629.0919799804688,
      "logps/rejected": -870.4136352539062,
      "loss": 0.3259,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.831345558166504,
      "rewards/margins": 2.3878016471862793,
      "rewards/rejected": -6.219147205352783,
      "step": 337
    },
    {
      "epoch": 0.7224151749933209,
      "grad_norm": 17.894860864055556,
      "learning_rate": 1.506804273910418e-07,
      "logits/chosen": -1.0389041900634766,
      "logits/rejected": -0.9967656135559082,
      "logps/chosen": -597.2425537109375,
      "logps/rejected": -690.931884765625,
      "loss": 0.3708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.6145212650299072,
      "rewards/margins": 1.04227876663208,
      "rewards/rejected": -4.656799793243408,
      "step": 338
    },
    {
      "epoch": 0.7245524979962596,
      "grad_norm": 24.07466390909236,
      "learning_rate": 1.485340270100143e-07,
      "logits/chosen": -0.8043147921562195,
      "logits/rejected": -0.8566476106643677,
      "logps/chosen": -721.7520751953125,
      "logps/rejected": -766.68896484375,
      "loss": 0.384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -4.88438081741333,
      "rewards/margins": 0.5339137315750122,
      "rewards/rejected": -5.418294906616211,
      "step": 339
    },
    {
      "epoch": 0.7266898209991985,
      "grad_norm": 23.369533031738897,
      "learning_rate": 1.4639889862548805e-07,
      "logits/chosen": -1.1091313362121582,
      "logits/rejected": -1.1004701852798462,
      "logps/chosen": -672.283447265625,
      "logps/rejected": -803.2059936523438,
      "loss": 0.4195,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.266483306884766,
      "rewards/margins": 1.2370930910110474,
      "rewards/rejected": -5.503575801849365,
      "step": 340
    },
    {
      "epoch": 0.7288271440021373,
      "grad_norm": 18.450003910880596,
      "learning_rate": 1.4427516169763443e-07,
      "logits/chosen": -0.9054552912712097,
      "logits/rejected": -0.9048701524734497,
      "logps/chosen": -747.39013671875,
      "logps/rejected": -927.0717163085938,
      "loss": 0.3928,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -5.073859214782715,
      "rewards/margins": 1.7517752647399902,
      "rewards/rejected": -6.825634479522705,
      "step": 341
    },
    {
      "epoch": 0.7309644670050761,
      "grad_norm": 23.60596160513393,
      "learning_rate": 1.4216293504927448e-07,
      "logits/chosen": -1.067272424697876,
      "logits/rejected": -1.0798040628433228,
      "logps/chosen": -676.371826171875,
      "logps/rejected": -869.149169921875,
      "loss": 0.379,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.598742961883545,
      "rewards/margins": 1.6217763423919678,
      "rewards/rejected": -6.220519065856934,
      "step": 342
    },
    {
      "epoch": 0.733101790008015,
      "grad_norm": 21.614156690463627,
      "learning_rate": 1.400623368592302e-07,
      "logits/chosen": -0.9705036878585815,
      "logits/rejected": -0.9520074725151062,
      "logps/chosen": -678.7489013671875,
      "logps/rejected": -816.748779296875,
      "loss": 0.4281,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.580175876617432,
      "rewards/margins": 1.3712455034255981,
      "rewards/rejected": -5.951421737670898,
      "step": 343
    },
    {
      "epoch": 0.7352391130109538,
      "grad_norm": 23.05746652103131,
      "learning_rate": 1.3797348465571306e-07,
      "logits/chosen": -0.9307321310043335,
      "logits/rejected": -0.9591343402862549,
      "logps/chosen": -746.435546875,
      "logps/rejected": -1008.3950805664062,
      "loss": 0.3567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.570581912994385,
      "rewards/margins": 2.517709255218506,
      "rewards/rejected": -7.088291168212891,
      "step": 344
    },
    {
      "epoch": 0.7373764360138926,
      "grad_norm": 18.10148284802051,
      "learning_rate": 1.3589649530974773e-07,
      "logits/chosen": -1.0869290828704834,
      "logits/rejected": -1.0850858688354492,
      "logps/chosen": -658.4027099609375,
      "logps/rejected": -982.6063842773438,
      "loss": 0.3781,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.2937188148498535,
      "rewards/margins": 2.9649014472961426,
      "rewards/rejected": -7.258620262145996,
      "step": 345
    },
    {
      "epoch": 0.7395137590168315,
      "grad_norm": 23.41128054527833,
      "learning_rate": 1.3383148502863378e-07,
      "logits/chosen": -1.0312128067016602,
      "logits/rejected": -1.1049691438674927,
      "logps/chosen": -814.4356689453125,
      "logps/rejected": -798.0429077148438,
      "loss": 0.514,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -5.450601577758789,
      "rewards/margins": 0.045907363295555115,
      "rewards/rejected": -5.496509552001953,
      "step": 346
    },
    {
      "epoch": 0.7416510820197703,
      "grad_norm": 18.50899288865318,
      "learning_rate": 1.317785693494433e-07,
      "logits/chosen": -1.0586178302764893,
      "logits/rejected": -1.029137134552002,
      "logps/chosen": -691.7874755859375,
      "logps/rejected": -813.152099609375,
      "loss": 0.3592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9073405265808105,
      "rewards/margins": 1.246219515800476,
      "rewards/rejected": -5.153559684753418,
      "step": 347
    },
    {
      "epoch": 0.743788405022709,
      "grad_norm": 20.32668011498763,
      "learning_rate": 1.2973786313255695e-07,
      "logits/chosen": -0.8740507960319519,
      "logits/rejected": -0.9160584211349487,
      "logps/chosen": -662.5859375,
      "logps/rejected": -873.8433227539062,
      "loss": 0.4097,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.367182731628418,
      "rewards/margins": 2.035503625869751,
      "rewards/rejected": -6.4026875495910645,
      "step": 348
    },
    {
      "epoch": 0.7459257280256478,
      "grad_norm": 23.02424339442235,
      "learning_rate": 1.2770948055523742e-07,
      "logits/chosen": -1.0587798357009888,
      "logits/rejected": -1.0669668912887573,
      "logps/chosen": -809.7626953125,
      "logps/rejected": -983.58984375,
      "loss": 0.3906,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.2913947105407715,
      "rewards/margins": 1.9067097902297974,
      "rewards/rejected": -7.1981048583984375,
      "step": 349
    },
    {
      "epoch": 0.7480630510285867,
      "grad_norm": 25.77622479036058,
      "learning_rate": 1.2569353510524156e-07,
      "logits/chosen": -1.069049596786499,
      "logits/rejected": -1.1337969303131104,
      "logps/chosen": -720.7489013671875,
      "logps/rejected": -938.7985229492188,
      "loss": 0.4103,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.965020179748535,
      "rewards/margins": 2.0392074584960938,
      "rewards/rejected": -7.004227638244629,
      "step": 350
    },
    {
      "epoch": 0.7502003740315255,
      "grad_norm": 17.60740131186996,
      "learning_rate": 1.2369013957447015e-07,
      "logits/chosen": -1.0574181079864502,
      "logits/rejected": -1.0570802688598633,
      "logps/chosen": -694.0639038085938,
      "logps/rejected": -829.4335327148438,
      "loss": 0.3407,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.506197452545166,
      "rewards/margins": 1.4661719799041748,
      "rewards/rejected": -5.97236967086792,
      "step": 351
    },
    {
      "epoch": 0.7523376970344643,
      "grad_norm": 17.276020504905794,
      "learning_rate": 1.216994060526577e-07,
      "logits/chosen": -1.1085654497146606,
      "logits/rejected": -1.0444343090057373,
      "logps/chosen": -699.8790893554688,
      "logps/rejected": -857.7581176757812,
      "loss": 0.3333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.61436128616333,
      "rewards/margins": 1.533206582069397,
      "rewards/rejected": -6.147567272186279,
      "step": 352
    },
    {
      "epoch": 0.7544750200374032,
      "grad_norm": 20.20530099751691,
      "learning_rate": 1.197214459211006e-07,
      "logits/chosen": -1.028286099433899,
      "logits/rejected": -1.0080163478851318,
      "logps/chosen": -729.8060913085938,
      "logps/rejected": -908.0136108398438,
      "loss": 0.3995,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.445433616638184,
      "rewards/margins": 1.915571689605713,
      "rewards/rejected": -6.361005783081055,
      "step": 353
    },
    {
      "epoch": 0.756612343040342,
      "grad_norm": 21.65892045821199,
      "learning_rate": 1.1775636984642612e-07,
      "logits/chosen": -1.001996636390686,
      "logits/rejected": -0.9989675283432007,
      "logps/chosen": -658.6278076171875,
      "logps/rejected": -824.7811279296875,
      "loss": 0.3989,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.4795989990234375,
      "rewards/margins": 1.533904790878296,
      "rewards/rejected": -6.0135040283203125,
      "step": 354
    },
    {
      "epoch": 0.7587496660432808,
      "grad_norm": 18.745090346580444,
      "learning_rate": 1.1580428777439973e-07,
      "logits/chosen": -1.1561853885650635,
      "logits/rejected": -1.0848287343978882,
      "logps/chosen": -622.0913696289062,
      "logps/rejected": -787.5831298828125,
      "loss": 0.3734,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.140936851501465,
      "rewards/margins": 1.524781346321106,
      "rewards/rejected": -5.665718078613281,
      "step": 355
    },
    {
      "epoch": 0.7608869890462197,
      "grad_norm": 18.598176758293135,
      "learning_rate": 1.1386530892377399e-07,
      "logits/chosen": -0.9880399107933044,
      "logits/rejected": -0.9888783097267151,
      "logps/chosen": -613.0370483398438,
      "logps/rejected": -777.3377075195312,
      "loss": 0.3762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9182024002075195,
      "rewards/margins": 1.6418043375015259,
      "rewards/rejected": -5.560007095336914,
      "step": 356
    },
    {
      "epoch": 0.7630243120491584,
      "grad_norm": 20.075874653172413,
      "learning_rate": 1.1193954178017814e-07,
      "logits/chosen": -0.9616910219192505,
      "logits/rejected": -1.0504767894744873,
      "logps/chosen": -782.3270874023438,
      "logps/rejected": -1025.231689453125,
      "loss": 0.4392,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.93320894241333,
      "rewards/margins": 2.5418636798858643,
      "rewards/rejected": -7.475072383880615,
      "step": 357
    },
    {
      "epoch": 0.7651616350520972,
      "grad_norm": 18.869618371734305,
      "learning_rate": 1.1002709409004803e-07,
      "logits/chosen": -1.1053738594055176,
      "logits/rejected": -1.0864824056625366,
      "logps/chosen": -768.8507080078125,
      "logps/rejected": -939.0321044921875,
      "loss": 0.3226,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.1586713790893555,
      "rewards/margins": 1.6820204257965088,
      "rewards/rejected": -6.840692043304443,
      "step": 358
    },
    {
      "epoch": 0.7672989580550361,
      "grad_norm": 22.39575999598468,
      "learning_rate": 1.0812807285459737e-07,
      "logits/chosen": -0.8523386120796204,
      "logits/rejected": -0.8684836030006409,
      "logps/chosen": -634.4469604492188,
      "logps/rejected": -910.783447265625,
      "loss": 0.4391,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.161304950714111,
      "rewards/margins": 2.574713945388794,
      "rewards/rejected": -6.736019134521484,
      "step": 359
    },
    {
      "epoch": 0.7694362810579749,
      "grad_norm": 24.655379460289804,
      "learning_rate": 1.0624258432383165e-07,
      "logits/chosen": -0.9855338931083679,
      "logits/rejected": -0.9286175966262817,
      "logps/chosen": -821.5938110351562,
      "logps/rejected": -976.9266357421875,
      "loss": 0.4425,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.609586715698242,
      "rewards/margins": 1.46958327293396,
      "rewards/rejected": -7.0791707038879395,
      "step": 360
    },
    {
      "epoch": 0.7715736040609137,
      "grad_norm": 23.07044684161756,
      "learning_rate": 1.0437073399060298e-07,
      "logits/chosen": -0.9206562638282776,
      "logits/rejected": -0.8095078468322754,
      "logps/chosen": -618.7409057617188,
      "logps/rejected": -815.9017333984375,
      "loss": 0.396,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.211564064025879,
      "rewards/margins": 1.776960849761963,
      "rewards/rejected": -5.988524913787842,
      "step": 361
    },
    {
      "epoch": 0.7737109270638525,
      "grad_norm": 20.255683251830362,
      "learning_rate": 1.0251262658470838e-07,
      "logits/chosen": -1.0115031003952026,
      "logits/rejected": -0.9697583317756653,
      "logps/chosen": -671.9834594726562,
      "logps/rejected": -856.5125122070312,
      "loss": 0.4114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.316826343536377,
      "rewards/margins": 1.731043815612793,
      "rewards/rejected": -6.04787015914917,
      "step": 362
    },
    {
      "epoch": 0.7758482500667914,
      "grad_norm": 36.27259699112537,
      "learning_rate": 1.0066836606702937e-07,
      "logits/chosen": -0.9073708057403564,
      "logits/rejected": -1.013731598854065,
      "logps/chosen": -687.530029296875,
      "logps/rejected": -816.8972778320312,
      "loss": 0.5571,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.503976821899414,
      "rewards/margins": 1.4232851266860962,
      "rewards/rejected": -5.927262306213379,
      "step": 363
    },
    {
      "epoch": 0.7779855730697302,
      "grad_norm": 18.487130214617952,
      "learning_rate": 9.883805562371581e-08,
      "logits/chosen": -1.096664547920227,
      "logits/rejected": -1.151119351387024,
      "logps/chosen": -627.2655029296875,
      "logps/rejected": -766.6741943359375,
      "loss": 0.3511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.355429649353027,
      "rewards/margins": 1.6355644464492798,
      "rewards/rejected": -5.990993976593018,
      "step": 364
    },
    {
      "epoch": 0.7801228960726689,
      "grad_norm": 17.467778162939954,
      "learning_rate": 9.702179766041299e-08,
      "logits/chosen": -1.1129746437072754,
      "logits/rejected": -1.0882641077041626,
      "logps/chosen": -609.0090942382812,
      "logps/rejected": -835.59375,
      "loss": 0.3284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.666825294494629,
      "rewards/margins": 2.0798864364624023,
      "rewards/rejected": -5.746711730957031,
      "step": 365
    },
    {
      "epoch": 0.7822602190756078,
      "grad_norm": 26.497135911369625,
      "learning_rate": 9.521969379653121e-08,
      "logits/chosen": -1.0410820245742798,
      "logits/rejected": -1.075594425201416,
      "logps/chosen": -861.631591796875,
      "logps/rejected": -1022.0664672851562,
      "loss": 0.4385,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.093762397766113,
      "rewards/margins": 1.7978633642196655,
      "rewards/rejected": -6.891625881195068,
      "step": 366
    },
    {
      "epoch": 0.7843975420785466,
      "grad_norm": 19.54124775721202,
      "learning_rate": 9.343184485956086e-08,
      "logits/chosen": -1.0408751964569092,
      "logits/rejected": -1.1065208911895752,
      "logps/chosen": -597.0819702148438,
      "logps/rejected": -755.0318603515625,
      "loss": 0.3985,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.7190897464752197,
      "rewards/margins": 1.8094736337661743,
      "rewards/rejected": -5.528563499450684,
      "step": 367
    },
    {
      "epoch": 0.7865348650814854,
      "grad_norm": 23.844802367349075,
      "learning_rate": 9.16583508794308e-08,
      "logits/chosen": -0.9044683575630188,
      "logits/rejected": -0.8596857786178589,
      "logps/chosen": -730.1447143554688,
      "logps/rejected": -878.2034912109375,
      "loss": 0.3,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.852618217468262,
      "rewards/margins": 1.4661897420883179,
      "rewards/rejected": -6.318808078765869,
      "step": 368
    },
    {
      "epoch": 0.7886721880844243,
      "grad_norm": 20.562190767658738,
      "learning_rate": 8.989931108291197e-08,
      "logits/chosen": -0.9048526287078857,
      "logits/rejected": -0.9187178611755371,
      "logps/chosen": -699.8194580078125,
      "logps/rejected": -844.4905395507812,
      "loss": 0.3635,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.6599884033203125,
      "rewards/margins": 1.4540899991989136,
      "rewards/rejected": -6.114078521728516,
      "step": 369
    },
    {
      "epoch": 0.7908095110873631,
      "grad_norm": 20.73718481631679,
      "learning_rate": 8.815482388806554e-08,
      "logits/chosen": -1.0599019527435303,
      "logits/rejected": -1.065873622894287,
      "logps/chosen": -760.4301147460938,
      "logps/rejected": -946.1746215820312,
      "loss": 0.4287,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.120336532592773,
      "rewards/margins": 1.838600993156433,
      "rewards/rejected": -6.958937168121338,
      "step": 370
    },
    {
      "epoch": 0.7929468340903019,
      "grad_norm": 30.164533991222857,
      "learning_rate": 8.642498689873619e-08,
      "logits/chosen": -1.0670329332351685,
      "logits/rejected": -1.1095917224884033,
      "logps/chosen": -768.016357421875,
      "logps/rejected": -985.7068481445312,
      "loss": 0.4338,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.948676586151123,
      "rewards/margins": 2.0501699447631836,
      "rewards/rejected": -6.998846530914307,
      "step": 371
    },
    {
      "epoch": 0.7950841570932408,
      "grad_norm": 18.533158040155175,
      "learning_rate": 8.47098968990914e-08,
      "logits/chosen": -1.212889313697815,
      "logits/rejected": -1.174148678779602,
      "logps/chosen": -575.9932250976562,
      "logps/rejected": -694.8380126953125,
      "loss": 0.3543,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.408289670944214,
      "rewards/margins": 1.2301357984542847,
      "rewards/rejected": -4.638425350189209,
      "step": 372
    },
    {
      "epoch": 0.7972214800961795,
      "grad_norm": 19.740449187608878,
      "learning_rate": 8.300964984820672e-08,
      "logits/chosen": -0.9624559879302979,
      "logits/rejected": -0.9103982448577881,
      "logps/chosen": -758.9552001953125,
      "logps/rejected": -952.0904541015625,
      "loss": 0.385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.812871932983398,
      "rewards/margins": 1.8450303077697754,
      "rewards/rejected": -6.657902240753174,
      "step": 373
    },
    {
      "epoch": 0.7993588030991183,
      "grad_norm": 17.857229527882023,
      "learning_rate": 8.13243408746961e-08,
      "logits/chosen": -1.044665813446045,
      "logits/rejected": -1.0194975137710571,
      "logps/chosen": -696.4033813476562,
      "logps/rejected": -845.8680419921875,
      "loss": 0.3787,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.486112594604492,
      "rewards/margins": 1.4695513248443604,
      "rewards/rejected": -5.955664157867432,
      "step": 374
    },
    {
      "epoch": 0.8014961261020572,
      "grad_norm": 27.402209380453634,
      "learning_rate": 7.96540642713902e-08,
      "logits/chosen": -1.0042835474014282,
      "logits/rejected": -0.9937084317207336,
      "logps/chosen": -798.3321533203125,
      "logps/rejected": -931.4829711914062,
      "loss": 0.4162,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -5.05294132232666,
      "rewards/margins": 1.302819848060608,
      "rewards/rejected": -6.35576057434082,
      "step": 375
    },
    {
      "epoch": 0.803633449104996,
      "grad_norm": 28.3566059208529,
      "learning_rate": 7.799891349006019e-08,
      "logits/chosen": -1.1246654987335205,
      "logits/rejected": -1.1276860237121582,
      "logps/chosen": -656.9721069335938,
      "logps/rejected": -813.6206665039062,
      "loss": 0.4719,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.687532901763916,
      "rewards/margins": 1.3982094526290894,
      "rewards/rejected": -6.085742473602295,
      "step": 376
    },
    {
      "epoch": 0.8057707721079348,
      "grad_norm": 18.856881189055258,
      "learning_rate": 7.635898113618957e-08,
      "logits/chosen": -1.2034432888031006,
      "logits/rejected": -1.1600438356399536,
      "logps/chosen": -585.2130737304688,
      "logps/rejected": -785.3953247070312,
      "loss": 0.3801,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.6061196327209473,
      "rewards/margins": 1.8955157995224,
      "rewards/rejected": -5.501636028289795,
      "step": 377
    },
    {
      "epoch": 0.8079080951108736,
      "grad_norm": 21.136594852281714,
      "learning_rate": 7.473435896379258e-08,
      "logits/chosen": -1.1049622297286987,
      "logits/rejected": -1.025770902633667,
      "logps/chosen": -697.8673706054688,
      "logps/rejected": -841.3628540039062,
      "loss": 0.4047,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.117053985595703,
      "rewards/margins": 1.336838960647583,
      "rewards/rejected": -5.453892707824707,
      "step": 378
    },
    {
      "epoch": 0.8100454181138125,
      "grad_norm": 21.622158153196573,
      "learning_rate": 7.312513787028065e-08,
      "logits/chosen": -1.018204927444458,
      "logits/rejected": -1.0209660530090332,
      "logps/chosen": -551.69091796875,
      "logps/rejected": -765.5928344726562,
      "loss": 0.3705,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.391167402267456,
      "rewards/margins": 1.9136013984680176,
      "rewards/rejected": -5.304769039154053,
      "step": 379
    },
    {
      "epoch": 0.8121827411167513,
      "grad_norm": 25.692774884542565,
      "learning_rate": 7.153140789137667e-08,
      "logits/chosen": -1.1156607866287231,
      "logits/rejected": -1.1090834140777588,
      "logps/chosen": -587.258056640625,
      "logps/rejected": -709.8887329101562,
      "loss": 0.417,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.669807195663452,
      "rewards/margins": 1.1046210527420044,
      "rewards/rejected": -4.774428367614746,
      "step": 380
    },
    {
      "epoch": 0.81432006411969,
      "grad_norm": 22.676599376198244,
      "learning_rate": 6.995325819607783e-08,
      "logits/chosen": -0.8035112619400024,
      "logits/rejected": -0.7905966639518738,
      "logps/chosen": -724.717529296875,
      "logps/rejected": -951.2332153320312,
      "loss": 0.3864,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.354774475097656,
      "rewards/margins": 2.2559542655944824,
      "rewards/rejected": -6.6107282638549805,
      "step": 381
    },
    {
      "epoch": 0.8164573871226289,
      "grad_norm": 20.060527238580573,
      "learning_rate": 6.839077708166608e-08,
      "logits/chosen": -0.9624785780906677,
      "logits/rejected": -0.9216483235359192,
      "logps/chosen": -759.6564331054688,
      "logps/rejected": -846.4254760742188,
      "loss": 0.4242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.706582546234131,
      "rewards/margins": 0.9559379816055298,
      "rewards/rejected": -5.662520408630371,
      "step": 382
    },
    {
      "epoch": 0.8185947101255677,
      "grad_norm": 20.450745536438863,
      "learning_rate": 6.684405196876843e-08,
      "logits/chosen": -0.9839971661567688,
      "logits/rejected": -0.9619154930114746,
      "logps/chosen": -659.962890625,
      "logps/rejected": -834.504638671875,
      "loss": 0.4453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.435713768005371,
      "rewards/margins": 1.5213212966918945,
      "rewards/rejected": -5.957034587860107,
      "step": 383
    },
    {
      "epoch": 0.8207320331285065,
      "grad_norm": 21.923861116889775,
      "learning_rate": 6.531316939646522e-08,
      "logits/chosen": -1.0496439933776855,
      "logits/rejected": -1.0298651456832886,
      "logps/chosen": -652.1452026367188,
      "logps/rejected": -761.5989990234375,
      "loss": 0.4117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.186628341674805,
      "rewards/margins": 1.098493218421936,
      "rewards/rejected": -5.285121917724609,
      "step": 384
    },
    {
      "epoch": 0.8228693561314454,
      "grad_norm": 15.872158871462359,
      "learning_rate": 6.379821501744902e-08,
      "logits/chosen": -0.9291130900382996,
      "logits/rejected": -0.9559027552604675,
      "logps/chosen": -637.105712890625,
      "logps/rejected": -784.9781494140625,
      "loss": 0.3618,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.092860698699951,
      "rewards/margins": 1.487864375114441,
      "rewards/rejected": -5.580724716186523,
      "step": 385
    },
    {
      "epoch": 0.8250066791343842,
      "grad_norm": 21.170940984738564,
      "learning_rate": 6.229927359323156e-08,
      "logits/chosen": -1.013567566871643,
      "logits/rejected": -0.9818376302719116,
      "logps/chosen": -581.1494140625,
      "logps/rejected": -750.385009765625,
      "loss": 0.3848,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5493507385253906,
      "rewards/margins": 1.4864931106567383,
      "rewards/rejected": -5.035844326019287,
      "step": 386
    },
    {
      "epoch": 0.827144002137323,
      "grad_norm": 20.203562100764046,
      "learning_rate": 6.081642898940186e-08,
      "logits/chosen": -0.9751057624816895,
      "logits/rejected": -0.9479618072509766,
      "logps/chosen": -535.7874145507812,
      "logps/rejected": -717.0790405273438,
      "loss": 0.3476,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4093728065490723,
      "rewards/margins": 1.679715871810913,
      "rewards/rejected": -5.089088439941406,
      "step": 387
    },
    {
      "epoch": 0.8292813251402619,
      "grad_norm": 16.226862763327308,
      "learning_rate": 5.9349764170933674e-08,
      "logits/chosen": -0.9501919746398926,
      "logits/rejected": -0.9817054867744446,
      "logps/chosen": -545.6821899414062,
      "logps/rejected": -637.3515625,
      "loss": 0.4066,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.451007843017578,
      "rewards/margins": 1.0472853183746338,
      "rewards/rejected": -4.498293399810791,
      "step": 388
    },
    {
      "epoch": 0.8314186481432007,
      "grad_norm": 21.301359722964037,
      "learning_rate": 5.789936119754407e-08,
      "logits/chosen": -1.070288062095642,
      "logits/rejected": -1.03058660030365,
      "logps/chosen": -773.905517578125,
      "logps/rejected": -884.7736206054688,
      "loss": 0.3989,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.410757064819336,
      "rewards/margins": 1.2033069133758545,
      "rewards/rejected": -5.6140642166137695,
      "step": 389
    },
    {
      "epoch": 0.8335559711461394,
      "grad_norm": 16.155956822127486,
      "learning_rate": 5.646530121910162e-08,
      "logits/chosen": -0.9560039043426514,
      "logits/rejected": -0.9677013158798218,
      "logps/chosen": -555.0410766601562,
      "logps/rejected": -647.4817504882812,
      "loss": 0.3702,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.367013931274414,
      "rewards/margins": 1.1322979927062988,
      "rewards/rejected": -4.499312400817871,
      "step": 390
    },
    {
      "epoch": 0.8356932941490782,
      "grad_norm": 17.445283127579135,
      "learning_rate": 5.5047664471086284e-08,
      "logits/chosen": -1.0367249250411987,
      "logits/rejected": -1.0191857814788818,
      "logps/chosen": -623.27978515625,
      "logps/rejected": -845.25048828125,
      "loss": 0.354,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.8465986251831055,
      "rewards/margins": 2.2664730548858643,
      "rewards/rejected": -6.113072395324707,
      "step": 391
    },
    {
      "epoch": 0.8378306171520171,
      "grad_norm": 17.389408198261865,
      "learning_rate": 5.3646530270100556e-08,
      "logits/chosen": -1.1824917793273926,
      "logits/rejected": -1.1786624193191528,
      "logps/chosen": -629.4869384765625,
      "logps/rejected": -910.8460693359375,
      "loss": 0.3307,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.6656405925750732,
      "rewards/margins": 2.664851188659668,
      "rewards/rejected": -6.33049201965332,
      "step": 392
    },
    {
      "epoch": 0.8399679401549559,
      "grad_norm": 16.107124952267284,
      "learning_rate": 5.2261977009431193e-08,
      "logits/chosen": -1.2528231143951416,
      "logits/rejected": -1.2414242029190063,
      "logps/chosen": -679.169189453125,
      "logps/rejected": -904.231689453125,
      "loss": 0.3437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.949631690979004,
      "rewards/margins": 1.9819270372390747,
      "rewards/rejected": -5.931559085845947,
      "step": 393
    },
    {
      "epoch": 0.8421052631578947,
      "grad_norm": 14.65787560767015,
      "learning_rate": 5.089408215466349e-08,
      "logits/chosen": -1.1920839548110962,
      "logits/rejected": -1.1330435276031494,
      "logps/chosen": -591.995849609375,
      "logps/rejected": -754.32421875,
      "loss": 0.3777,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.461061954498291,
      "rewards/margins": 1.4271600246429443,
      "rewards/rejected": -4.8882222175598145,
      "step": 394
    },
    {
      "epoch": 0.8442425861608336,
      "grad_norm": 19.973781830012914,
      "learning_rate": 4.9542922239346865e-08,
      "logits/chosen": -0.9532175660133362,
      "logits/rejected": -0.9305418729782104,
      "logps/chosen": -700.0613403320312,
      "logps/rejected": -854.81201171875,
      "loss": 0.4104,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.186321258544922,
      "rewards/margins": 1.5199509859085083,
      "rewards/rejected": -5.706272602081299,
      "step": 395
    },
    {
      "epoch": 0.8463799091637724,
      "grad_norm": 26.08541321380337,
      "learning_rate": 4.8208572860713086e-08,
      "logits/chosen": -1.149269461631775,
      "logits/rejected": -1.1293056011199951,
      "logps/chosen": -618.3247680664062,
      "logps/rejected": -777.4251708984375,
      "loss": 0.4851,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4766008853912354,
      "rewards/margins": 1.5825449228286743,
      "rewards/rejected": -5.059146404266357,
      "step": 396
    },
    {
      "epoch": 0.8485172321667112,
      "grad_norm": 29.339785563765055,
      "learning_rate": 4.689110867544645e-08,
      "logits/chosen": -1.015437126159668,
      "logits/rejected": -0.9925183057785034,
      "logps/chosen": -679.9268188476562,
      "logps/rejected": -870.37890625,
      "loss": 0.344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.385658264160156,
      "rewards/margins": 1.9707388877868652,
      "rewards/rejected": -6.35639762878418,
      "step": 397
    },
    {
      "epoch": 0.85065455516965,
      "grad_norm": 19.625508223329977,
      "learning_rate": 4.5590603395506626e-08,
      "logits/chosen": -1.0667216777801514,
      "logits/rejected": -1.049599289894104,
      "logps/chosen": -589.4276123046875,
      "logps/rejected": -712.3489379882812,
      "loss": 0.4014,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.779334306716919,
      "rewards/margins": 1.0566929578781128,
      "rewards/rejected": -4.836027145385742,
      "step": 398
    },
    {
      "epoch": 0.8527918781725888,
      "grad_norm": 19.959692153012657,
      "learning_rate": 4.430712978400462e-08,
      "logits/chosen": -1.0223028659820557,
      "logits/rejected": -1.0172353982925415,
      "logps/chosen": -494.1994323730469,
      "logps/rejected": -618.9550170898438,
      "loss": 0.4284,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.038055658340454,
      "rewards/margins": 1.1104681491851807,
      "rewards/rejected": -4.148523807525635,
      "step": 399
    },
    {
      "epoch": 0.8549292011755276,
      "grad_norm": 22.852143621544176,
      "learning_rate": 4.304075965113171e-08,
      "logits/chosen": -1.2095826864242554,
      "logits/rejected": -1.1936359405517578,
      "logps/chosen": -572.5497436523438,
      "logps/rejected": -702.556396484375,
      "loss": 0.3827,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.307962656021118,
      "rewards/margins": 1.31619131565094,
      "rewards/rejected": -4.624154090881348,
      "step": 400
    },
    {
      "epoch": 0.8549292011755276,
      "eval_logits/chosen": -1.128338098526001,
      "eval_logits/rejected": -1.1495376825332642,
      "eval_logps/chosen": -710.064697265625,
      "eval_logps/rejected": -863.7308349609375,
      "eval_loss": 0.3871653079986572,
      "eval_rewards/accuracies": 0.8709677457809448,
      "eval_rewards/chosen": -4.3693461418151855,
      "eval_rewards/margins": 1.5976247787475586,
      "eval_rewards/rejected": -5.966970920562744,
      "eval_runtime": 127.1231,
      "eval_samples_per_second": 15.426,
      "eval_steps_per_second": 0.488,
      "step": 400
    },
    {
      "epoch": 0.8570665241784665,
      "grad_norm": 20.918890009764137,
      "learning_rate": 4.1791563850141695e-08,
      "logits/chosen": -0.9988707900047302,
      "logits/rejected": -1.0228999853134155,
      "logps/chosen": -705.0687255859375,
      "logps/rejected": -993.6966552734375,
      "loss": 0.3109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.109606742858887,
      "rewards/margins": 2.890212297439575,
      "rewards/rejected": -6.999818801879883,
      "step": 401
    },
    {
      "epoch": 0.8592038471814053,
      "grad_norm": 22.50167745931539,
      "learning_rate": 4.0559612273386616e-08,
      "logits/chosen": -1.098199725151062,
      "logits/rejected": -1.1250154972076416,
      "logps/chosen": -785.855224609375,
      "logps/rejected": -917.9451904296875,
      "loss": 0.3891,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.712646484375,
      "rewards/margins": 1.341720700263977,
      "rewards/rejected": -6.0543670654296875,
      "step": 402
    },
    {
      "epoch": 0.8613411701843441,
      "grad_norm": 16.355069952426618,
      "learning_rate": 3.934497384840607e-08,
      "logits/chosen": -0.9095240235328674,
      "logits/rejected": -0.8507342338562012,
      "logps/chosen": -621.5723876953125,
      "logps/rejected": -791.5116577148438,
      "loss": 0.4111,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.8870487213134766,
      "rewards/margins": 1.6334927082061768,
      "rewards/rejected": -5.520541667938232,
      "step": 403
    },
    {
      "epoch": 0.863478493187283,
      "grad_norm": 20.03294448892495,
      "learning_rate": 3.8147716534071274e-08,
      "logits/chosen": -1.0887154340744019,
      "logits/rejected": -1.095017671585083,
      "logps/chosen": -605.3096923828125,
      "logps/rejected": -669.5009155273438,
      "loss": 0.4435,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -3.492586374282837,
      "rewards/margins": 0.6767375469207764,
      "rewards/rejected": -4.169323921203613,
      "step": 404
    },
    {
      "epoch": 0.8656158161902218,
      "grad_norm": 19.638131692176458,
      "learning_rate": 3.696790731678217e-08,
      "logits/chosen": -1.0262949466705322,
      "logits/rejected": -1.0150538682937622,
      "logps/chosen": -797.56689453125,
      "logps/rejected": -929.2540893554688,
      "loss": 0.4095,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.939427852630615,
      "rewards/margins": 1.520028829574585,
      "rewards/rejected": -6.459456920623779,
      "step": 405
    },
    {
      "epoch": 0.8677531391931605,
      "grad_norm": 19.64316736105873,
      "learning_rate": 3.5805612206719885e-08,
      "logits/chosen": -0.9092820882797241,
      "logits/rejected": -0.9483243227005005,
      "logps/chosen": -681.32958984375,
      "logps/rejected": -848.2050170898438,
      "loss": 0.349,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.447098255157471,
      "rewards/margins": 1.769409418106079,
      "rewards/rejected": -6.216507911682129,
      "step": 406
    },
    {
      "epoch": 0.8698904621960993,
      "grad_norm": 16.2039010339986,
      "learning_rate": 3.466089623415333e-08,
      "logits/chosen": -0.9941847324371338,
      "logits/rejected": -0.9800211787223816,
      "logps/chosen": -583.2901000976562,
      "logps/rejected": -831.3629150390625,
      "loss": 0.3707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.81510329246521,
      "rewards/margins": 2.43589448928833,
      "rewards/rejected": -6.250997543334961,
      "step": 407
    },
    {
      "epoch": 0.8720277851990382,
      "grad_norm": 17.593791491624575,
      "learning_rate": 3.3533823445800965e-08,
      "logits/chosen": -1.0939316749572754,
      "logits/rejected": -1.0538439750671387,
      "logps/chosen": -563.0457763671875,
      "logps/rejected": -702.2050170898438,
      "loss": 0.3908,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.7340145111083984,
      "rewards/margins": 1.5319364070892334,
      "rewards/rejected": -5.2659502029418945,
      "step": 408
    },
    {
      "epoch": 0.874165108201977,
      "grad_norm": 17.753144065442136,
      "learning_rate": 3.242445690124729e-08,
      "logits/chosen": -1.0609853267669678,
      "logits/rejected": -1.0725332498550415,
      "logps/chosen": -745.48046875,
      "logps/rejected": -863.9573974609375,
      "loss": 0.3536,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.6711931228637695,
      "rewards/margins": 1.1300690174102783,
      "rewards/rejected": -5.801262378692627,
      "step": 409
    },
    {
      "epoch": 0.8763024312049158,
      "grad_norm": 22.5406397148522,
      "learning_rate": 3.133285866941445e-08,
      "logits/chosen": -0.9890793561935425,
      "logits/rejected": -1.017478585243225,
      "logps/chosen": -489.8434143066406,
      "logps/rejected": -649.6760864257812,
      "loss": 0.463,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.189424991607666,
      "rewards/margins": 1.429877758026123,
      "rewards/rejected": -4.619302749633789,
      "step": 410
    },
    {
      "epoch": 0.8784397542078547,
      "grad_norm": 16.06885831310049,
      "learning_rate": 3.025908982508966e-08,
      "logits/chosen": -1.04689359664917,
      "logits/rejected": -0.9806681871414185,
      "logps/chosen": -767.8428344726562,
      "logps/rejected": -1035.4447021484375,
      "loss": 0.3126,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.467133045196533,
      "rewards/margins": 2.5061354637145996,
      "rewards/rejected": -6.973268985748291,
      "step": 411
    },
    {
      "epoch": 0.8805770772107935,
      "grad_norm": 19.662285734659836,
      "learning_rate": 2.9203210445508332e-08,
      "logits/chosen": -1.0549553632736206,
      "logits/rejected": -1.008999228477478,
      "logps/chosen": -739.88720703125,
      "logps/rejected": -918.3170166015625,
      "loss": 0.4546,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.464229583740234,
      "rewards/margins": 1.5558303594589233,
      "rewards/rejected": -6.0200605392456055,
      "step": 412
    },
    {
      "epoch": 0.8827144002137323,
      "grad_norm": 27.990757995621482,
      "learning_rate": 2.8165279606992298e-08,
      "logits/chosen": -0.9684846997261047,
      "logits/rejected": -0.9231318235397339,
      "logps/chosen": -719.86083984375,
      "logps/rejected": -912.4309692382812,
      "loss": 0.3979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.8175811767578125,
      "rewards/margins": 1.912165641784668,
      "rewards/rejected": -6.729746341705322,
      "step": 413
    },
    {
      "epoch": 0.8848517232166712,
      "grad_norm": 17.729091244130924,
      "learning_rate": 2.7145355381644793e-08,
      "logits/chosen": -1.0500560998916626,
      "logits/rejected": -1.0563499927520752,
      "logps/chosen": -557.0240478515625,
      "logps/rejected": -717.523681640625,
      "loss": 0.3655,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.6325652599334717,
      "rewards/margins": 1.5925527811050415,
      "rewards/rejected": -5.225118160247803,
      "step": 414
    },
    {
      "epoch": 0.88698904621961,
      "grad_norm": 19.8351911008517,
      "learning_rate": 2.6143494834101153e-08,
      "logits/chosen": -0.9061897993087769,
      "logits/rejected": -0.9052097797393799,
      "logps/chosen": -704.2842407226562,
      "logps/rejected": -743.07958984375,
      "loss": 0.3664,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.4450836181640625,
      "rewards/margins": 0.44577062129974365,
      "rewards/rejected": -4.890854835510254,
      "step": 415
    },
    {
      "epoch": 0.8891263692225487,
      "grad_norm": 37.85808985299137,
      "learning_rate": 2.5159754018336403e-08,
      "logits/chosen": -1.1449795961380005,
      "logits/rejected": -1.1638754606246948,
      "logps/chosen": -737.4818115234375,
      "logps/rejected": -989.4339599609375,
      "loss": 0.4523,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.679790019989014,
      "rewards/margins": 2.3331949710845947,
      "rewards/rejected": -7.012984275817871,
      "step": 416
    },
    {
      "epoch": 0.8912636922254876,
      "grad_norm": 21.579885808296115,
      "learning_rate": 2.4194187974528553e-08,
      "logits/chosen": -1.104061484336853,
      "logits/rejected": -1.0894217491149902,
      "logps/chosen": -679.447509765625,
      "logps/rejected": -833.4044189453125,
      "loss": 0.3835,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.530219554901123,
      "rewards/margins": 1.5757548809051514,
      "rewards/rejected": -6.105974197387695,
      "step": 417
    },
    {
      "epoch": 0.8934010152284264,
      "grad_norm": 18.32956314016428,
      "learning_rate": 2.3246850725979427e-08,
      "logits/chosen": -1.0385122299194336,
      "logits/rejected": -1.0385818481445312,
      "logps/chosen": -666.721435546875,
      "logps/rejected": -787.4942626953125,
      "loss": 0.3856,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.093487739562988,
      "rewards/margins": 1.256799340248108,
      "rewards/rejected": -5.350286960601807,
      "step": 418
    },
    {
      "epoch": 0.8955383382313652,
      "grad_norm": 26.323713896853675,
      "learning_rate": 2.2317795276091977e-08,
      "logits/chosen": -0.9605960249900818,
      "logits/rejected": -0.9666808247566223,
      "logps/chosen": -626.7743530273438,
      "logps/rejected": -742.9835205078125,
      "loss": 0.4861,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.809014320373535,
      "rewards/margins": 1.3489842414855957,
      "rewards/rejected": -5.157999038696289,
      "step": 419
    },
    {
      "epoch": 0.897675661234304,
      "grad_norm": 29.92038686319786,
      "learning_rate": 2.1407073605404862e-08,
      "logits/chosen": -1.0402368307113647,
      "logits/rejected": -0.9794477224349976,
      "logps/chosen": -743.0745849609375,
      "logps/rejected": -964.0947875976562,
      "loss": 0.4477,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.6502366065979,
      "rewards/margins": 2.061614990234375,
      "rewards/rejected": -6.711851119995117,
      "step": 420
    },
    {
      "epoch": 0.8998129842372429,
      "grad_norm": 18.598312258529717,
      "learning_rate": 2.0514736668683917e-08,
      "logits/chosen": -1.1911952495574951,
      "logits/rejected": -1.2088751792907715,
      "logps/chosen": -688.5134887695312,
      "logps/rejected": -876.8580932617188,
      "loss": 0.4087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.259193420410156,
      "rewards/margins": 1.567337155342102,
      "rewards/rejected": -5.826530456542969,
      "step": 421
    },
    {
      "epoch": 0.9019503072401817,
      "grad_norm": 23.213199916317794,
      "learning_rate": 1.964083439207135e-08,
      "logits/chosen": -1.1022056341171265,
      "logits/rejected": -1.1523065567016602,
      "logps/chosen": -729.16259765625,
      "logps/rejected": -873.5059204101562,
      "loss": 0.4305,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.697969913482666,
      "rewards/margins": 1.660890817642212,
      "rewards/rejected": -6.358860969543457,
      "step": 422
    },
    {
      "epoch": 0.9040876302431204,
      "grad_norm": 17.78006990070099,
      "learning_rate": 1.878541567029251e-08,
      "logits/chosen": -1.0770941972732544,
      "logits/rejected": -1.0126092433929443,
      "logps/chosen": -606.9859008789062,
      "logps/rejected": -791.86474609375,
      "loss": 0.3209,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.224216461181641,
      "rewards/margins": 1.6672827005386353,
      "rewards/rejected": -5.891498565673828,
      "step": 423
    },
    {
      "epoch": 0.9062249532460593,
      "grad_norm": 28.8620946540648,
      "learning_rate": 1.794852836392009e-08,
      "logits/chosen": -0.9167647957801819,
      "logits/rejected": -0.9531383514404297,
      "logps/chosen": -565.0890502929688,
      "logps/rejected": -742.736328125,
      "loss": 0.4255,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4048359394073486,
      "rewards/margins": 1.6988798379898071,
      "rewards/rejected": -5.103715896606445,
      "step": 424
    },
    {
      "epoch": 0.9083622762489981,
      "grad_norm": 20.499467136576563,
      "learning_rate": 1.7130219296696264e-08,
      "logits/chosen": -0.9955915212631226,
      "logits/rejected": -1.0456228256225586,
      "logps/chosen": -657.0418090820312,
      "logps/rejected": -762.1922607421875,
      "loss": 0.3629,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.7994446754455566,
      "rewards/margins": 1.2025730609893799,
      "rewards/rejected": -5.002017974853516,
      "step": 425
    },
    {
      "epoch": 0.9104995992519369,
      "grad_norm": 17.206845074940055,
      "learning_rate": 1.6330534252913086e-08,
      "logits/chosen": -1.0955368280410767,
      "logits/rejected": -1.081055998802185,
      "logps/chosen": -706.4891967773438,
      "logps/rejected": -900.5104370117188,
      "loss": 0.3426,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.797817230224609,
      "rewards/margins": 1.9311976432800293,
      "rewards/rejected": -6.729015350341797,
      "step": 426
    },
    {
      "epoch": 0.9126369222548758,
      "grad_norm": 17.119036708717413,
      "learning_rate": 1.5549517974850724e-08,
      "logits/chosen": -0.8513895273208618,
      "logits/rejected": -0.8118205070495605,
      "logps/chosen": -611.9838256835938,
      "logps/rejected": -755.0986328125,
      "loss": 0.3786,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.1998372077941895,
      "rewards/margins": 1.362562656402588,
      "rewards/rejected": -5.562399864196777,
      "step": 427
    },
    {
      "epoch": 0.9147742452578146,
      "grad_norm": 30.666142274697034,
      "learning_rate": 1.4787214160274341e-08,
      "logits/chosen": -1.1465550661087036,
      "logits/rejected": -1.1435879468917847,
      "logps/chosen": -637.793701171875,
      "logps/rejected": -803.086669921875,
      "loss": 0.4319,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.044920444488525,
      "rewards/margins": 1.6718565225601196,
      "rewards/rejected": -5.7167768478393555,
      "step": 428
    },
    {
      "epoch": 0.9169115682607534,
      "grad_norm": 17.479325335479295,
      "learning_rate": 1.4043665459988868e-08,
      "logits/chosen": -1.1191812753677368,
      "logits/rejected": -1.0620485544204712,
      "logps/chosen": -617.3558349609375,
      "logps/rejected": -858.5902099609375,
      "loss": 0.3978,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.054196357727051,
      "rewards/margins": 2.3324999809265137,
      "rewards/rejected": -6.3866963386535645,
      "step": 429
    },
    {
      "epoch": 0.9190488912636923,
      "grad_norm": 19.870927489532274,
      "learning_rate": 1.3318913475452925e-08,
      "logits/chosen": -1.1520562171936035,
      "logits/rejected": -1.0596178770065308,
      "logps/chosen": -827.845947265625,
      "logps/rejected": -1057.0477294921875,
      "loss": 0.3633,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.0968828201293945,
      "rewards/margins": 2.2637462615966797,
      "rewards/rejected": -7.360629081726074,
      "step": 430
    },
    {
      "epoch": 0.921186214266631,
      "grad_norm": 19.001198221145234,
      "learning_rate": 1.2612998756451366e-08,
      "logits/chosen": -1.0448460578918457,
      "logits/rejected": -1.0036524534225464,
      "logps/chosen": -565.2469482421875,
      "logps/rejected": -768.9799194335938,
      "loss": 0.3764,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.4013609886169434,
      "rewards/margins": 2.0993523597717285,
      "rewards/rejected": -5.500713348388672,
      "step": 431
    },
    {
      "epoch": 0.9233235372695698,
      "grad_norm": 24.185158250695462,
      "learning_rate": 1.192596079882613e-08,
      "logits/chosen": -1.016557216644287,
      "logits/rejected": -0.9917696714401245,
      "logps/chosen": -754.6041259765625,
      "logps/rejected": -1039.5848388671875,
      "loss": 0.4467,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.575927257537842,
      "rewards/margins": 2.5993194580078125,
      "rewards/rejected": -7.175246715545654,
      "step": 432
    },
    {
      "epoch": 0.9254608602725087,
      "grad_norm": 18.630615195275038,
      "learning_rate": 1.1257838042266754e-08,
      "logits/chosen": -0.9740123152732849,
      "logits/rejected": -0.9422295689582825,
      "logps/chosen": -684.7318115234375,
      "logps/rejected": -853.54931640625,
      "loss": 0.3922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.472279071807861,
      "rewards/margins": 1.5442816019058228,
      "rewards/rejected": -6.016561031341553,
      "step": 433
    },
    {
      "epoch": 0.9275981832754475,
      "grad_norm": 19.666494529888574,
      "learning_rate": 1.0608667868159592e-08,
      "logits/chosen": -1.0905375480651855,
      "logits/rejected": -1.0876115560531616,
      "logps/chosen": -828.9867553710938,
      "logps/rejected": -1100.01318359375,
      "loss": 0.3426,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.9841694831848145,
      "rewards/margins": 2.305903911590576,
      "rewards/rejected": -7.290073394775391,
      "step": 434
    },
    {
      "epoch": 0.9297355062783863,
      "grad_norm": 18.138692746396494,
      "learning_rate": 9.978486597496372e-09,
      "logits/chosen": -0.9997820854187012,
      "logits/rejected": -0.997348427772522,
      "logps/chosen": -644.03955078125,
      "logps/rejected": -883.5596923828125,
      "loss": 0.3441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.433194160461426,
      "rewards/margins": 2.0547845363616943,
      "rewards/rejected": -6.487978935241699,
      "step": 435
    },
    {
      "epoch": 0.9318728292813251,
      "grad_norm": 22.806909647633304,
      "learning_rate": 9.367329488842001e-09,
      "logits/chosen": -1.0480362176895142,
      "logits/rejected": -1.025497317314148,
      "logps/chosen": -719.160400390625,
      "logps/rejected": -870.7994384765625,
      "loss": 0.3881,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.514778137207031,
      "rewards/margins": 1.344374656677246,
      "rewards/rejected": -5.859152317047119,
      "step": 436
    },
    {
      "epoch": 0.934010152284264,
      "grad_norm": 34.30034026086993,
      "learning_rate": 8.775230736361733e-09,
      "logits/chosen": -0.9019148349761963,
      "logits/rejected": -1.021255612373352,
      "logps/chosen": -610.4000244140625,
      "logps/rejected": -666.5303344726562,
      "loss": 0.5228,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -3.8257713317871094,
      "rewards/margins": 0.6458057165145874,
      "rewards/rejected": -4.471577167510986,
      "step": 437
    },
    {
      "epoch": 0.9361474752872028,
      "grad_norm": 26.008895134527737,
      "learning_rate": 8.202223467908299e-09,
      "logits/chosen": -1.208720088005066,
      "logits/rejected": -1.1576639413833618,
      "logps/chosen": -704.7356567382812,
      "logps/rejected": -935.2647094726562,
      "loss": 0.429,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.226789474487305,
      "rewards/margins": 2.07527494430542,
      "rewards/rejected": -6.302064418792725,
      "step": 438
    },
    {
      "epoch": 0.9382847982901416,
      "grad_norm": 23.1664224383978,
      "learning_rate": 7.64833974316801e-09,
      "logits/chosen": -0.9366050958633423,
      "logits/rejected": -1.0036077499389648,
      "logps/chosen": -692.1240844726562,
      "logps/rejected": -962.029541015625,
      "loss": 0.3745,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.578640937805176,
      "rewards/margins": 2.751880407333374,
      "rewards/rejected": -7.330521583557129,
      "step": 439
    },
    {
      "epoch": 0.9404221212930804,
      "grad_norm": 25.822513994850507,
      "learning_rate": 7.113610551867583e-09,
      "logits/chosen": -1.1607967615127563,
      "logits/rejected": -1.162611484527588,
      "logps/chosen": -727.8070678710938,
      "logps/rejected": -895.8046875,
      "loss": 0.4345,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.304068565368652,
      "rewards/margins": 1.6857025623321533,
      "rewards/rejected": -5.989770889282227,
      "step": 440
    },
    {
      "epoch": 0.9425594442960192,
      "grad_norm": 24.174938576336224,
      "learning_rate": 6.5980658120396106e-09,
      "logits/chosen": -1.013208270072937,
      "logits/rejected": -0.9189801216125488,
      "logps/chosen": -747.7554321289062,
      "logps/rejected": -1023.7532348632812,
      "loss": 0.4307,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.824457168579102,
      "rewards/margins": 2.6137688159942627,
      "rewards/rejected": -7.438226222991943,
      "step": 441
    },
    {
      "epoch": 0.944696767298958,
      "grad_norm": 21.66924362068404,
      "learning_rate": 6.101734368349104e-09,
      "logits/chosen": -1.089997410774231,
      "logits/rejected": -1.1180981397628784,
      "logps/chosen": -580.3588256835938,
      "logps/rejected": -653.803955078125,
      "loss": 0.3454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.4357962608337402,
      "rewards/margins": 0.8316761255264282,
      "rewards/rejected": -4.267472267150879,
      "step": 442
    },
    {
      "epoch": 0.9468340903018969,
      "grad_norm": 18.92185198276508,
      "learning_rate": 5.624643990479616e-09,
      "logits/chosen": -0.9731349349021912,
      "logits/rejected": -1.0123114585876465,
      "logps/chosen": -744.20068359375,
      "logps/rejected": -880.1783447265625,
      "loss": 0.4027,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.492010116577148,
      "rewards/margins": 1.5610084533691406,
      "rewards/rejected": -6.053019046783447,
      "step": 443
    },
    {
      "epoch": 0.9489714133048357,
      "grad_norm": 19.40472791239129,
      "learning_rate": 5.166821371579277e-09,
      "logits/chosen": -0.8690105676651001,
      "logits/rejected": -0.9122138023376465,
      "logps/chosen": -592.9273071289062,
      "logps/rejected": -692.8623046875,
      "loss": 0.3917,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.9664011001586914,
      "rewards/margins": 1.128931999206543,
      "rewards/rejected": -5.095333099365234,
      "step": 444
    },
    {
      "epoch": 0.9511087363077745,
      "grad_norm": 17.606366912659396,
      "learning_rate": 4.7282921267675306e-09,
      "logits/chosen": -1.1353975534439087,
      "logits/rejected": -1.069037914276123,
      "logps/chosen": -617.16845703125,
      "logps/rejected": -833.6134033203125,
      "loss": 0.3496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9898786544799805,
      "rewards/margins": 1.8373284339904785,
      "rewards/rejected": -5.827206611633301,
      "step": 445
    },
    {
      "epoch": 0.9532460593107134,
      "grad_norm": 22.334810036861253,
      "learning_rate": 4.309080791701819e-09,
      "logits/chosen": -0.9705511927604675,
      "logits/rejected": -0.9864068031311035,
      "logps/chosen": -772.231689453125,
      "logps/rejected": -924.7553100585938,
      "loss": 0.3883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.6818671226501465,
      "rewards/margins": 1.767922043800354,
      "rewards/rejected": -6.449788570404053,
      "step": 446
    },
    {
      "epoch": 0.9553833823136522,
      "grad_norm": 18.928358142495153,
      "learning_rate": 3.909210821205016e-09,
      "logits/chosen": -1.0940232276916504,
      "logits/rejected": -1.1222940683364868,
      "logps/chosen": -569.94482421875,
      "logps/rejected": -690.92578125,
      "loss": 0.3731,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.7058024406433105,
      "rewards/margins": 1.2121186256408691,
      "rewards/rejected": -4.91792106628418,
      "step": 447
    },
    {
      "epoch": 0.957520705316591,
      "grad_norm": 19.387388769733043,
      "learning_rate": 3.528704587952963e-09,
      "logits/chosen": -0.9721835255622864,
      "logits/rejected": -0.9633288383483887,
      "logps/chosen": -691.5062255859375,
      "logps/rejected": -864.804443359375,
      "loss": 0.3357,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.21799898147583,
      "rewards/margins": 1.6493499279022217,
      "rewards/rejected": -5.867349147796631,
      "step": 448
    },
    {
      "epoch": 0.9596580283195298,
      "grad_norm": 15.631110311383365,
      "learning_rate": 3.1675833812228547e-09,
      "logits/chosen": -1.143801212310791,
      "logits/rejected": -1.1312353610992432,
      "logps/chosen": -591.1502685546875,
      "logps/rejected": -746.0208129882812,
      "loss": 0.4032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.8137049674987793,
      "rewards/margins": 1.4380216598510742,
      "rewards/rejected": -5.2517266273498535,
      "step": 449
    },
    {
      "epoch": 0.9617953513224686,
      "grad_norm": 23.63954938708458,
      "learning_rate": 2.825867405701826e-09,
      "logits/chosen": -1.0871686935424805,
      "logits/rejected": -1.0616105794906616,
      "logps/chosen": -785.9561157226562,
      "logps/rejected": -949.911376953125,
      "loss": 0.4622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.555358409881592,
      "rewards/margins": 1.6683781147003174,
      "rewards/rejected": -6.22373628616333,
      "step": 450
    },
    {
      "epoch": 0.9639326743254074,
      "grad_norm": 22.04528865685746,
      "learning_rate": 2.503575780356926e-09,
      "logits/chosen": -1.1157352924346924,
      "logits/rejected": -1.122366189956665,
      "logps/chosen": -687.6873779296875,
      "logps/rejected": -789.6235961914062,
      "loss": 0.4453,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.202605724334717,
      "rewards/margins": 1.0868072509765625,
      "rewards/rejected": -5.289412975311279,
      "step": 451
    },
    {
      "epoch": 0.9660699973283462,
      "grad_norm": 20.620417690774662,
      "learning_rate": 2.2007265373650885e-09,
      "logits/chosen": -1.181131362915039,
      "logits/rejected": -1.1398506164550781,
      "logps/chosen": -752.2744750976562,
      "logps/rejected": -903.8162231445312,
      "loss": 0.4466,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.831580638885498,
      "rewards/margins": 1.5455623865127563,
      "rewards/rejected": -6.377142906188965,
      "step": 452
    },
    {
      "epoch": 0.9682073203312851,
      "grad_norm": 21.990068975057824,
      "learning_rate": 1.9173366211043483e-09,
      "logits/chosen": -1.066049337387085,
      "logits/rejected": -1.145350694656372,
      "logps/chosen": -732.5936889648438,
      "logps/rejected": -820.995361328125,
      "loss": 0.4411,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -4.793266296386719,
      "rewards/margins": 0.9948696494102478,
      "rewards/rejected": -5.788136005401611,
      "step": 453
    },
    {
      "epoch": 0.9703446433342239,
      "grad_norm": 18.52277369573019,
      "learning_rate": 1.653421887205747e-09,
      "logits/chosen": -1.002540111541748,
      "logits/rejected": -1.0223251581192017,
      "logps/chosen": -650.927978515625,
      "logps/rejected": -728.2699584960938,
      "loss": 0.3655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.248347759246826,
      "rewards/margins": 0.7103351354598999,
      "rewards/rejected": -4.958683013916016,
      "step": 454
    },
    {
      "epoch": 0.9724819663371627,
      "grad_norm": 33.05665306914434,
      "learning_rate": 1.408997101666326e-09,
      "logits/chosen": -1.1706193685531616,
      "logits/rejected": -1.108201265335083,
      "logps/chosen": -693.7764282226562,
      "logps/rejected": -911.11962890625,
      "loss": 0.4464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.211030006408691,
      "rewards/margins": 1.9548094272613525,
      "rewards/rejected": -6.165839195251465,
      "step": 455
    },
    {
      "epoch": 0.9746192893401016,
      "grad_norm": 18.73958161040263,
      "learning_rate": 1.1840759400229328e-09,
      "logits/chosen": -0.9501774907112122,
      "logits/rejected": -0.9423767924308777,
      "logps/chosen": -872.7874755859375,
      "logps/rejected": -1080.9110107421875,
      "loss": 0.4383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.329258441925049,
      "rewards/margins": 1.8498148918151855,
      "rewards/rejected": -7.179073810577393,
      "step": 456
    },
    {
      "epoch": 0.9767566123430403,
      "grad_norm": 23.633155328899434,
      "learning_rate": 9.786709865869547e-10,
      "logits/chosen": -1.0224264860153198,
      "logits/rejected": -0.9899497032165527,
      "logps/chosen": -691.5697631835938,
      "logps/rejected": -1032.654052734375,
      "loss": 0.3647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.561187744140625,
      "rewards/margins": 3.524183750152588,
      "rewards/rejected": -8.085371971130371,
      "step": 457
    },
    {
      "epoch": 0.9788939353459791,
      "grad_norm": 20.02815998548278,
      "learning_rate": 7.927937337403711e-10,
      "logits/chosen": -1.1305997371673584,
      "logits/rejected": -1.1589406728744507,
      "logps/chosen": -720.5179443359375,
      "logps/rejected": -914.910400390625,
      "loss": 0.413,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.451896667480469,
      "rewards/margins": 2.01759672164917,
      "rewards/rejected": -6.4694929122924805,
      "step": 458
    },
    {
      "epoch": 0.981031258348918,
      "grad_norm": 21.2779508066291,
      "learning_rate": 6.264545812926958e-10,
      "logits/chosen": -1.0647788047790527,
      "logits/rejected": -1.044015884399414,
      "logps/chosen": -644.20263671875,
      "logps/rejected": -890.7341918945312,
      "loss": 0.4476,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.727494239807129,
      "rewards/margins": 2.3570847511291504,
      "rewards/rejected": -6.0845794677734375,
      "step": 459
    },
    {
      "epoch": 0.9831685813518568,
      "grad_norm": 19.021711845408657,
      "learning_rate": 4.796628358991583e-10,
      "logits/chosen": -1.0134469270706177,
      "logits/rejected": -1.0272549390792847,
      "logps/chosen": -688.755859375,
      "logps/rejected": -888.3029174804688,
      "loss": 0.3752,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.3303303718566895,
      "rewards/margins": 1.7046260833740234,
      "rewards/rejected": -6.034955978393555,
      "step": 460
    },
    {
      "epoch": 0.9853059043547956,
      "grad_norm": 22.296101394440072,
      "learning_rate": 3.524267105398937e-10,
      "logits/chosen": -1.0911214351654053,
      "logits/rejected": -1.1538598537445068,
      "logps/chosen": -693.211181640625,
      "logps/rejected": -855.1487426757812,
      "loss": 0.3781,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.124536514282227,
      "rewards/margins": 1.6871302127838135,
      "rewards/rejected": -5.811666965484619,
      "step": 461
    },
    {
      "epoch": 0.9874432273577345,
      "grad_norm": 18.18277828641047,
      "learning_rate": 2.447533240604871e-10,
      "logits/chosen": -1.0473253726959229,
      "logits/rejected": -1.0253962278366089,
      "logps/chosen": -531.8089599609375,
      "logps/rejected": -790.6080322265625,
      "loss": 0.3867,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4056613445281982,
      "rewards/margins": 2.266097068786621,
      "rewards/rejected": -5.671758651733398,
      "step": 462
    },
    {
      "epoch": 0.9895805503606733,
      "grad_norm": 20.702683706379894,
      "learning_rate": 1.5664870077383707e-10,
      "logits/chosen": -1.0570118427276611,
      "logits/rejected": -1.0660746097564697,
      "logps/chosen": -655.458251953125,
      "logps/rejected": -789.9827880859375,
      "loss": 0.311,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.035057544708252,
      "rewards/margins": 1.403611183166504,
      "rewards/rejected": -5.438669204711914,
      "step": 463
    },
    {
      "epoch": 0.9917178733636121,
      "grad_norm": 30.473984400030304,
      "learning_rate": 8.811777012271427e-11,
      "logits/chosen": -1.2154146432876587,
      "logits/rejected": -1.2058013677597046,
      "logps/chosen": -639.0227661132812,
      "logps/rejected": -820.2091064453125,
      "loss": 0.3856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.121957778930664,
      "rewards/margins": 1.6997212171554565,
      "rewards/rejected": -5.821678638458252,
      "step": 464
    },
    {
      "epoch": 0.9938551963665508,
      "grad_norm": 23.82578403366403,
      "learning_rate": 3.9164366404298435e-11,
      "logits/chosen": -1.0110024213790894,
      "logits/rejected": -1.0080432891845703,
      "logps/chosen": -551.5545654296875,
      "logps/rejected": -713.6527099609375,
      "loss": 0.3884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.4868009090423584,
      "rewards/margins": 1.5603687763214111,
      "rewards/rejected": -5.047170162200928,
      "step": 465
    },
    {
      "epoch": 0.9959925193694897,
      "grad_norm": 23.209923292158702,
      "learning_rate": 9.791228555566799e-12,
      "logits/chosen": -0.9966657161712646,
      "logits/rejected": -1.022011160850525,
      "logps/chosen": -511.430908203125,
      "logps/rejected": -595.9152221679688,
      "loss": 0.397,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.4004158973693848,
      "rewards/margins": 0.7674331068992615,
      "rewards/rejected": -4.167849063873291,
      "step": 466
    },
    {
      "epoch": 0.9981298423724285,
      "grad_norm": 21.729353783070525,
      "learning_rate": 0.0,
      "logits/chosen": -1.0969784259796143,
      "logits/rejected": -1.0713554620742798,
      "logps/chosen": -821.9149780273438,
      "logps/rejected": -1118.57958984375,
      "loss": 0.3546,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.988923072814941,
      "rewards/margins": 3.1074531078338623,
      "rewards/rejected": -8.096376419067383,
      "step": 467
    },
    {
      "epoch": 0.9981298423724285,
      "step": 467,
      "total_flos": 0.0,
      "train_loss": 0.4754098514673286,
      "train_runtime": 9903.4254,
      "train_samples_per_second": 6.046,
      "train_steps_per_second": 0.047
    }
  ],
  "logging_steps": 1,
  "max_steps": 467,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}