{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 100,
  "global_step": 2907,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.7182130584192438e-09,
      "logits/chosen": -2.498650312423706,
      "logits/rejected": -2.6091811656951904,
      "logps/chosen": -96.33251953125,
      "logps/rejected": -92.86735534667969,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 1.718213058419244e-08,
      "logits/chosen": -3.101959228515625,
      "logits/rejected": -2.9548017978668213,
      "logps/chosen": -224.08120727539062,
      "logps/rejected": -199.21075439453125,
      "loss": 0.6915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005350981839001179,
      "rewards/margins": 0.0050353375263512135,
      "rewards/rejected": 0.000315645185764879,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 3.436426116838488e-08,
      "logits/chosen": -2.9904165267944336,
      "logits/rejected": -2.987039566040039,
      "logps/chosen": -278.6884765625,
      "logps/rejected": -236.29263305664062,
      "loss": 0.6904,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007055492140352726,
      "rewards/margins": 0.013753985986113548,
      "rewards/rejected": -0.006698492914438248,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 5.154639175257731e-08,
      "logits/chosen": -3.097510814666748,
      "logits/rejected": -3.1187407970428467,
      "logps/chosen": -298.73443603515625,
      "logps/rejected": -228.46926879882812,
      "loss": 0.6756,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.02428005076944828,
      "rewards/margins": 0.059538763016462326,
      "rewards/rejected": -0.035258710384368896,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 6.872852233676976e-08,
      "logits/chosen": -3.0823495388031006,
      "logits/rejected": -3.0725135803222656,
      "logps/chosen": -384.5050354003906,
      "logps/rejected": -236.9876708984375,
      "loss": 0.6507,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.1019597053527832,
      "rewards/margins": 0.17022760212421417,
      "rewards/rejected": -0.06826789677143097,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 8.59106529209622e-08,
      "logits/chosen": -2.94450306892395,
      "logits/rejected": -2.8841047286987305,
      "logps/chosen": -278.482421875,
      "logps/rejected": -227.1036834716797,
      "loss": 0.6306,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.06523530930280685,
      "rewards/margins": 0.1872607171535492,
      "rewards/rejected": -0.12202541530132294,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.0309278350515462e-07,
      "logits/chosen": -2.963169574737549,
      "logits/rejected": -2.883527994155884,
      "logps/chosen": -304.78021240234375,
      "logps/rejected": -283.13482666015625,
      "loss": 0.6023,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0056328559294342995,
      "rewards/margins": 0.2135467529296875,
      "rewards/rejected": -0.21917958557605743,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.202749140893471e-07,
      "logits/chosen": -2.9630908966064453,
      "logits/rejected": -2.825876235961914,
      "logps/chosen": -303.33544921875,
      "logps/rejected": -160.35809326171875,
      "loss": 0.5848,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.044426899403333664,
      "rewards/margins": 0.4234214723110199,
      "rewards/rejected": -0.3789946138858795,
      "step": 70
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.3745704467353952e-07,
      "logits/chosen": -3.0682828426361084,
      "logits/rejected": -2.9316720962524414,
      "logps/chosen": -249.2676239013672,
      "logps/rejected": -224.609619140625,
      "loss": 0.5516,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.14790545403957367,
      "rewards/margins": 0.6019492149353027,
      "rewards/rejected": -0.4540437161922455,
      "step": 80
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.5463917525773197e-07,
      "logits/chosen": -3.0987212657928467,
      "logits/rejected": -3.0108442306518555,
      "logps/chosen": -216.3074188232422,
      "logps/rejected": -206.27590942382812,
      "loss": 0.5071,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.19666776061058044,
      "rewards/margins": 1.0184780359268188,
      "rewards/rejected": -0.821810245513916,
      "step": 90
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.718213058419244e-07,
      "logits/chosen": -3.091062068939209,
      "logits/rejected": -2.9374005794525146,
      "logps/chosen": -314.8296203613281,
      "logps/rejected": -176.48611450195312,
      "loss": 0.5386,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.22487859427928925,
      "rewards/margins": 0.7823053598403931,
      "rewards/rejected": -0.5574267506599426,
      "step": 100
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -3.054979085922241,
      "eval_logits/rejected": -3.0135693550109863,
      "eval_logps/chosen": -254.21786499023438,
      "eval_logps/rejected": -204.1269073486328,
      "eval_loss": 0.5208388566970825,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": 0.05637993663549423,
      "eval_rewards/margins": 0.8084549307823181,
      "eval_rewards/rejected": -0.7520750761032104,
      "eval_runtime": 42.4363,
      "eval_samples_per_second": 47.129,
      "eval_steps_per_second": 0.377,
      "step": 100
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.8900343642611682e-07,
      "logits/chosen": -2.8181204795837402,
      "logits/rejected": -2.7204036712646484,
      "logps/chosen": -312.5907897949219,
      "logps/rejected": -214.4507293701172,
      "loss": 0.547,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1282820850610733,
      "rewards/margins": 0.89714515209198,
      "rewards/rejected": -1.025427222251892,
      "step": 110
    },
    {
      "epoch": 0.12,
      "learning_rate": 2.0618556701030925e-07,
      "logits/chosen": -2.9315900802612305,
      "logits/rejected": -2.943131923675537,
      "logps/chosen": -306.6047668457031,
      "logps/rejected": -288.87103271484375,
      "loss": 0.5276,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.17532110214233398,
      "rewards/margins": 0.4478148818016052,
      "rewards/rejected": -0.6231359839439392,
      "step": 120
    },
    {
      "epoch": 0.13,
      "learning_rate": 2.2336769759450173e-07,
      "logits/chosen": -3.0673885345458984,
      "logits/rejected": -2.946303606033325,
      "logps/chosen": -288.3648986816406,
      "logps/rejected": -202.46240234375,
      "loss": 0.5242,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.5023866891860962,
      "rewards/margins": 1.2662431001663208,
      "rewards/rejected": -0.7638564705848694,
      "step": 130
    },
    {
      "epoch": 0.14,
      "learning_rate": 2.405498281786942e-07,
      "logits/chosen": -2.935978412628174,
      "logits/rejected": -2.9787638187408447,
      "logps/chosen": -225.0258331298828,
      "logps/rejected": -197.45791625976562,
      "loss": 0.4987,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.20422056317329407,
      "rewards/margins": 0.8866212964057922,
      "rewards/rejected": -0.6824007630348206,
      "step": 140
    },
    {
      "epoch": 0.15,
      "learning_rate": 2.5773195876288655e-07,
      "logits/chosen": -2.891925573348999,
      "logits/rejected": -3.011976480484009,
      "logps/chosen": -375.8110046386719,
      "logps/rejected": -204.3079376220703,
      "loss": 0.5238,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01360023021697998,
      "rewards/margins": 0.7446385025978088,
      "rewards/rejected": -0.7582387924194336,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.7491408934707903e-07,
      "logits/chosen": -3.1294617652893066,
      "logits/rejected": -3.121156692504883,
      "logps/chosen": -448.22021484375,
      "logps/rejected": -326.61553955078125,
      "loss": 0.4881,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.09536926448345184,
      "rewards/margins": 1.3267143964767456,
      "rewards/rejected": -1.4220836162567139,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 2.9209621993127146e-07,
      "logits/chosen": -2.906979560852051,
      "logits/rejected": -2.9241394996643066,
      "logps/chosen": -279.4745178222656,
      "logps/rejected": -243.3614501953125,
      "loss": 0.5082,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1525634229183197,
      "rewards/margins": 0.8320217132568359,
      "rewards/rejected": -0.984585165977478,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 3.0927835051546394e-07,
      "logits/chosen": -3.15456485748291,
      "logits/rejected": -3.120079517364502,
      "logps/chosen": -283.2970886230469,
      "logps/rejected": -260.8895568847656,
      "loss": 0.4729,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04973051697015762,
      "rewards/margins": 0.9862112998962402,
      "rewards/rejected": -1.0359418392181396,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 3.2646048109965636e-07,
      "logits/chosen": -3.1225669384002686,
      "logits/rejected": -3.066546678543091,
      "logps/chosen": -312.17938232421875,
      "logps/rejected": -226.7977752685547,
      "loss": 0.5136,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.060521602630615234,
      "rewards/margins": 1.1039137840270996,
      "rewards/rejected": -1.0433921813964844,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 3.436426116838488e-07,
      "logits/chosen": -3.0691170692443848,
      "logits/rejected": -3.0096538066864014,
      "logps/chosen": -269.39080810546875,
      "logps/rejected": -207.4473114013672,
      "loss": 0.4931,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.38108959794044495,
      "rewards/margins": 0.5996788740158081,
      "rewards/rejected": -0.9807685017585754,
      "step": 200
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -3.140662908554077,
      "eval_logits/rejected": -3.105587959289551,
      "eval_logps/chosen": -254.91360473632812,
      "eval_logps/rejected": -209.2888946533203,
      "eval_loss": 0.4881684482097626,
      "eval_rewards/accuracies": 0.78125,
      "eval_rewards/chosen": -0.013194993138313293,
      "eval_rewards/margins": 1.2550796270370483,
      "eval_rewards/rejected": -1.2682745456695557,
      "eval_runtime": 43.0581,
      "eval_samples_per_second": 46.449,
      "eval_steps_per_second": 0.372,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 3.608247422680412e-07,
      "logits/chosen": -3.2085659503936768,
      "logits/rejected": -3.0949082374572754,
      "logps/chosen": -386.8721008300781,
      "logps/rejected": -247.3134307861328,
      "loss": 0.4477,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13939574360847473,
      "rewards/margins": 1.158414602279663,
      "rewards/rejected": -1.2978103160858154,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.7800687285223364e-07,
      "logits/chosen": -2.9630062580108643,
      "logits/rejected": -2.9715027809143066,
      "logps/chosen": -173.3898468017578,
      "logps/rejected": -256.95513916015625,
      "loss": 0.5044,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14368362724781036,
      "rewards/margins": 0.8015314340591431,
      "rewards/rejected": -0.945215106010437,
      "step": 220
    },
    {
      "epoch": 0.24,
      "learning_rate": 3.9518900343642607e-07,
      "logits/chosen": -2.9683520793914795,
      "logits/rejected": -3.0452380180358887,
      "logps/chosen": -347.9017028808594,
      "logps/rejected": -276.4244079589844,
      "loss": 0.516,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.010433333925902843,
      "rewards/margins": 1.247714877128601,
      "rewards/rejected": -1.2581483125686646,
      "step": 230
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.123711340206185e-07,
      "logits/chosen": -3.1391446590423584,
      "logits/rejected": -3.1013171672821045,
      "logps/chosen": -374.7413024902344,
      "logps/rejected": -270.0039367675781,
      "loss": 0.4653,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08338235318660736,
      "rewards/margins": 0.9544317126274109,
      "rewards/rejected": -1.0378139019012451,
      "step": 240
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.2955326460481097e-07,
      "logits/chosen": -3.069174289703369,
      "logits/rejected": -2.962937831878662,
      "logps/chosen": -285.2818908691406,
      "logps/rejected": -281.72015380859375,
      "loss": 0.4746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09429286420345306,
      "rewards/margins": 1.3244707584381104,
      "rewards/rejected": -1.4187636375427246,
      "step": 250
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.4673539518900345e-07,
      "logits/chosen": -2.8843207359313965,
      "logits/rejected": -2.8770415782928467,
      "logps/chosen": -211.5966033935547,
      "logps/rejected": -242.7068328857422,
      "loss": 0.5133,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.43082481622695923,
      "rewards/margins": 1.3341903686523438,
      "rewards/rejected": -1.7650152444839478,
      "step": 260
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.639175257731959e-07,
      "logits/chosen": -3.0725252628326416,
      "logits/rejected": -2.915465831756592,
      "logps/chosen": -245.35208129882812,
      "logps/rejected": -256.63916015625,
      "loss": 0.5273,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32778286933898926,
      "rewards/margins": 1.0782946348190308,
      "rewards/rejected": -1.40607750415802,
      "step": 270
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.810996563573884e-07,
      "logits/chosen": -3.064030885696411,
      "logits/rejected": -3.022933006286621,
      "logps/chosen": -342.7411804199219,
      "logps/rejected": -256.2796936035156,
      "loss": 0.5139,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.2716239094734192,
      "rewards/margins": 1.2219887971878052,
      "rewards/rejected": -1.4936127662658691,
      "step": 280
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.982817869415807e-07,
      "logits/chosen": -3.0413384437561035,
      "logits/rejected": -2.969083070755005,
      "logps/chosen": -289.0894470214844,
      "logps/rejected": -221.3031463623047,
      "loss": 0.5064,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3141915202140808,
      "rewards/margins": 1.3370590209960938,
      "rewards/rejected": -1.6512506008148193,
      "step": 290
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.982798165137615e-07,
      "logits/chosen": -2.8817899227142334,
      "logits/rejected": -2.924848794937134,
      "logps/chosen": -220.947021484375,
      "logps/rejected": -265.0274658203125,
      "loss": 0.479,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5559160113334656,
      "rewards/margins": 0.9888324737548828,
      "rewards/rejected": -1.5447485446929932,
      "step": 300
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -3.13276743888855,
      "eval_logits/rejected": -3.0809109210968018,
      "eval_logps/chosen": -255.81625366210938,
      "eval_logps/rejected": -210.61862182617188,
      "eval_loss": 0.5038384795188904,
      "eval_rewards/accuracies": 0.78125,
      "eval_rewards/chosen": -0.10346230119466782,
      "eval_rewards/margins": 1.297783613204956,
      "eval_rewards/rejected": -1.4012458324432373,
      "eval_runtime": 42.7501,
      "eval_samples_per_second": 46.784,
      "eval_steps_per_second": 0.374,
      "step": 300
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.963685015290519e-07,
      "logits/chosen": -3.099226951599121,
      "logits/rejected": -2.9154460430145264,
      "logps/chosen": -370.572998046875,
      "logps/rejected": -260.42279052734375,
      "loss": 0.5327,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08727478235960007,
      "rewards/margins": 1.1489875316619873,
      "rewards/rejected": -1.0617127418518066,
      "step": 310
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.944571865443424e-07,
      "logits/chosen": -2.9195303916931152,
      "logits/rejected": -2.822550058364868,
      "logps/chosen": -222.0548553466797,
      "logps/rejected": -160.9757080078125,
      "loss": 0.4637,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.027837419882416725,
      "rewards/margins": 1.3940068483352661,
      "rewards/rejected": -1.421844244003296,
      "step": 320
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.92545871559633e-07,
      "logits/chosen": -3.008601188659668,
      "logits/rejected": -3.1163241863250732,
      "logps/chosen": -331.88385009765625,
      "logps/rejected": -305.1783142089844,
      "loss": 0.5029,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007843529805541039,
      "rewards/margins": 0.9688261151313782,
      "rewards/rejected": -0.960982620716095,
      "step": 330
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.906345565749235e-07,
      "logits/chosen": -2.98130464553833,
      "logits/rejected": -2.839319944381714,
      "logps/chosen": -245.6178741455078,
      "logps/rejected": -155.1374969482422,
      "loss": 0.5331,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3822270333766937,
      "rewards/margins": 0.9632396697998047,
      "rewards/rejected": -1.3454667329788208,
      "step": 340
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.88723241590214e-07,
      "logits/chosen": -2.945957899093628,
      "logits/rejected": -2.7072620391845703,
      "logps/chosen": -395.33709716796875,
      "logps/rejected": -254.1487274169922,
      "loss": 0.4787,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.023961830884218216,
      "rewards/margins": 1.7189300060272217,
      "rewards/rejected": -1.6949679851531982,
      "step": 350
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.868119266055046e-07,
      "logits/chosen": -2.8966143131256104,
      "logits/rejected": -2.9680020809173584,
      "logps/chosen": -226.3685760498047,
      "logps/rejected": -311.12890625,
      "loss": 0.49,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.773408055305481,
      "rewards/margins": 1.1420116424560547,
      "rewards/rejected": -1.9154198169708252,
      "step": 360
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.849006116207951e-07,
      "logits/chosen": -3.101799964904785,
      "logits/rejected": -2.958263874053955,
      "logps/chosen": -321.9137878417969,
      "logps/rejected": -279.8211364746094,
      "loss": 0.5009,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.48245924711227417,
      "rewards/margins": 1.1618621349334717,
      "rewards/rejected": -1.6443214416503906,
      "step": 370
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.829892966360856e-07,
      "logits/chosen": -3.0152487754821777,
      "logits/rejected": -2.990954637527466,
      "logps/chosen": -284.4853820800781,
      "logps/rejected": -275.23974609375,
      "loss": 0.5344,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.22872860729694366,
      "rewards/margins": 1.912491798400879,
      "rewards/rejected": -1.6837631464004517,
      "step": 380
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.810779816513762e-07,
      "logits/chosen": -3.0833258628845215,
      "logits/rejected": -3.007781982421875,
      "logps/chosen": -208.7422332763672,
      "logps/rejected": -203.84396362304688,
      "loss": 0.5237,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5863692760467529,
      "rewards/margins": 1.30623459815979,
      "rewards/rejected": -1.8926036357879639,
      "step": 390
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.791666666666667e-07,
      "logits/chosen": -3.0465681552886963,
      "logits/rejected": -2.8939976692199707,
      "logps/chosen": -295.1468811035156,
      "logps/rejected": -208.3681182861328,
      "loss": 0.5052,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.09109281003475189,
      "rewards/margins": 1.9571037292480469,
      "rewards/rejected": -1.8660109043121338,
      "step": 400
    },
    {
      "epoch": 0.41,
      "eval_logits/chosen": -2.9643757343292236,
      "eval_logits/rejected": -2.9103996753692627,
      "eval_logps/chosen": -256.70428466796875,
      "eval_logps/rejected": -215.38914489746094,
      "eval_loss": 0.5154134631156921,
      "eval_rewards/accuracies": 0.796875,
      "eval_rewards/chosen": -0.19226306676864624,
      "eval_rewards/margins": 1.6860344409942627,
      "eval_rewards/rejected": -1.8782974481582642,
      "eval_runtime": 42.6041,
      "eval_samples_per_second": 46.944,
      "eval_steps_per_second": 0.376,
      "step": 400
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.772553516819572e-07,
      "logits/chosen": -2.90427827835083,
      "logits/rejected": -2.846705675125122,
      "logps/chosen": -133.34495544433594,
      "logps/rejected": -192.99078369140625,
      "loss": 0.5246,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8221723437309265,
      "rewards/margins": 0.5950149297714233,
      "rewards/rejected": -1.417187213897705,
      "step": 410
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.753440366972477e-07,
      "logits/chosen": -3.146049976348877,
      "logits/rejected": -3.0406510829925537,
      "logps/chosen": -287.80450439453125,
      "logps/rejected": -296.94378662109375,
      "loss": 0.5322,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.45591288805007935,
      "rewards/margins": 0.4107814431190491,
      "rewards/rejected": -0.8666942715644836,
      "step": 420
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.7343272171253825e-07,
      "logits/chosen": -2.9290690422058105,
      "logits/rejected": -2.854919910430908,
      "logps/chosen": -266.37091064453125,
      "logps/rejected": -265.41650390625,
      "loss": 0.5384,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8489448428153992,
      "rewards/margins": 1.4326685667037964,
      "rewards/rejected": -2.28161358833313,
      "step": 430
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.715214067278288e-07,
      "logits/chosen": -2.983694076538086,
      "logits/rejected": -2.8835182189941406,
      "logps/chosen": -241.74203491210938,
      "logps/rejected": -232.90682983398438,
      "loss": 0.4691,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07152794301509857,
      "rewards/margins": 1.2356297969818115,
      "rewards/rejected": -1.3071579933166504,
      "step": 440
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.696100917431192e-07,
      "logits/chosen": -3.0933094024658203,
      "logits/rejected": -2.934081554412842,
      "logps/chosen": -261.3104553222656,
      "logps/rejected": -273.6822509765625,
      "loss": 0.5041,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.0025750964414328337,
      "rewards/margins": 1.066935658454895,
      "rewards/rejected": -1.064360499382019,
      "step": 450
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.6769877675840974e-07,
      "logits/chosen": -2.9390621185302734,
      "logits/rejected": -2.9396934509277344,
      "logps/chosen": -233.00534057617188,
      "logps/rejected": -216.82998657226562,
      "loss": 0.49,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.2104685753583908,
      "rewards/margins": 1.035353660583496,
      "rewards/rejected": -1.2458221912384033,
      "step": 460
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.6578746177370027e-07,
      "logits/chosen": -3.032357692718506,
      "logits/rejected": -2.9270567893981934,
      "logps/chosen": -294.4527893066406,
      "logps/rejected": -158.98008728027344,
      "loss": 0.4567,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.18893463909626007,
      "rewards/margins": 1.4661831855773926,
      "rewards/rejected": -1.6551177501678467,
      "step": 470
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.638761467889908e-07,
      "logits/chosen": -2.896667957305908,
      "logits/rejected": -2.9291975498199463,
      "logps/chosen": -211.07479858398438,
      "logps/rejected": -226.3457489013672,
      "loss": 0.6474,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.218103289604187,
      "rewards/margins": 1.827314019203186,
      "rewards/rejected": -2.045417308807373,
      "step": 480
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.6196483180428133e-07,
      "logits/chosen": -2.929793119430542,
      "logits/rejected": -2.8522984981536865,
      "logps/chosen": -258.91400146484375,
      "logps/rejected": -175.8672637939453,
      "loss": 0.5046,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1329614520072937,
      "rewards/margins": 1.482403039932251,
      "rewards/rejected": -1.615364670753479,
      "step": 490
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.600535168195718e-07,
      "logits/chosen": -2.7829298973083496,
      "logits/rejected": -2.8831605911254883,
      "logps/chosen": -204.31448364257812,
      "logps/rejected": -230.2319793701172,
      "loss": 0.4513,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.32916760444641113,
      "rewards/margins": 0.9852824211120605,
      "rewards/rejected": -1.3144500255584717,
      "step": 500
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -3.0656890869140625,
      "eval_logits/rejected": -3.0061020851135254,
      "eval_logps/chosen": -254.57415771484375,
      "eval_logps/rejected": -213.1681671142578,
      "eval_loss": 0.49787724018096924,
      "eval_rewards/accuracies": 0.796875,
      "eval_rewards/chosen": 0.02074863389134407,
      "eval_rewards/margins": 1.6769486665725708,
      "eval_rewards/rejected": -1.6561999320983887,
      "eval_runtime": 42.8458,
      "eval_samples_per_second": 46.679,
      "eval_steps_per_second": 0.373,
      "step": 500
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.5814220183486234e-07,
      "logits/chosen": -2.8923087120056152,
      "logits/rejected": -2.8640570640563965,
      "logps/chosen": -368.39154052734375,
      "logps/rejected": -338.0500793457031,
      "loss": 0.5188,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.49800175428390503,
      "rewards/margins": 1.331751823425293,
      "rewards/rejected": -1.8297535181045532,
      "step": 510
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.562308868501529e-07,
      "logits/chosen": -2.9221696853637695,
      "logits/rejected": -2.7648494243621826,
      "logps/chosen": -258.91290283203125,
      "logps/rejected": -301.06298828125,
      "loss": 0.5025,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4446481168270111,
      "rewards/margins": 1.80561101436615,
      "rewards/rejected": -2.2502589225769043,
      "step": 520
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.543195718654434e-07,
      "logits/chosen": -2.8445491790771484,
      "logits/rejected": -2.7915587425231934,
      "logps/chosen": -322.16131591796875,
      "logps/rejected": -219.9912567138672,
      "loss": 0.5031,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5867956876754761,
      "rewards/margins": 0.7803589105606079,
      "rewards/rejected": -1.367154598236084,
      "step": 530
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.5240825688073394e-07,
      "logits/chosen": -3.095763683319092,
      "logits/rejected": -2.850975513458252,
      "logps/chosen": -260.27935791015625,
      "logps/rejected": -244.0723419189453,
      "loss": 0.5143,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.33804479241371155,
      "rewards/margins": 1.2927238941192627,
      "rewards/rejected": -1.6307685375213623,
      "step": 540
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.504969418960244e-07,
      "logits/chosen": -3.1108663082122803,
      "logits/rejected": -2.967893123626709,
      "logps/chosen": -323.0700378417969,
      "logps/rejected": -306.6070251464844,
      "loss": 0.5103,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5680254101753235,
      "rewards/margins": 1.1552374362945557,
      "rewards/rejected": -1.7232627868652344,
      "step": 550
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.4858562691131495e-07,
      "logits/chosen": -3.050872325897217,
      "logits/rejected": -2.9496700763702393,
      "logps/chosen": -264.33990478515625,
      "logps/rejected": -256.26849365234375,
      "loss": 0.5017,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7357984781265259,
      "rewards/margins": 0.9639188647270203,
      "rewards/rejected": -1.6997172832489014,
      "step": 560
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.466743119266055e-07,
      "logits/chosen": -2.926020383834839,
      "logits/rejected": -2.8725242614746094,
      "logps/chosen": -343.10809326171875,
      "logps/rejected": -339.7952880859375,
      "loss": 0.4638,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.068613052368164,
      "rewards/margins": 0.6752510070800781,
      "rewards/rejected": -1.743863821029663,
      "step": 570
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.44762996941896e-07,
      "logits/chosen": -2.9404196739196777,
      "logits/rejected": -2.867061138153076,
      "logps/chosen": -307.84832763671875,
      "logps/rejected": -239.6619110107422,
      "loss": 0.4982,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06973789632320404,
      "rewards/margins": 2.0842127799987793,
      "rewards/rejected": -2.1539506912231445,
      "step": 580
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.4285168195718655e-07,
      "logits/chosen": -2.954078435897827,
      "logits/rejected": -2.972050189971924,
      "logps/chosen": -240.4013214111328,
      "logps/rejected": -257.27490234375,
      "loss": 0.4751,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3049514889717102,
      "rewards/margins": 1.0277297496795654,
      "rewards/rejected": -1.33268141746521,
      "step": 590
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.40940366972477e-07,
      "logits/chosen": -3.0263705253601074,
      "logits/rejected": -2.9350860118865967,
      "logps/chosen": -196.01318359375,
      "logps/rejected": -212.79452514648438,
      "loss": 0.4905,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.42911848425865173,
      "rewards/margins": 0.917349636554718,
      "rewards/rejected": -1.346468210220337,
      "step": 600
    },
    {
      "epoch": 0.62,
      "eval_logits/chosen": -3.016953468322754,
      "eval_logits/rejected": -2.93742299079895,
      "eval_logps/chosen": -255.72560119628906,
      "eval_logps/rejected": -212.45274353027344,
      "eval_loss": 0.4907330274581909,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -0.09439453482627869,
      "eval_rewards/margins": 1.4902656078338623,
      "eval_rewards/rejected": -1.584660291671753,
      "eval_runtime": 42.0993,
      "eval_samples_per_second": 47.507,
      "eval_steps_per_second": 0.38,
      "step": 600
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.3902905198776756e-07,
      "logits/chosen": -3.165318250656128,
      "logits/rejected": -2.9908506870269775,
      "logps/chosen": -407.2035217285156,
      "logps/rejected": -344.78912353515625,
      "loss": 0.4841,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.24180400371551514,
      "rewards/margins": 1.6740745306015015,
      "rewards/rejected": -1.9158786535263062,
      "step": 610
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.371177370030581e-07,
      "logits/chosen": -3.012429714202881,
      "logits/rejected": -2.929919958114624,
      "logps/chosen": -232.5025177001953,
      "logps/rejected": -212.27389526367188,
      "loss": 0.4751,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3227713704109192,
      "rewards/margins": 1.730791687965393,
      "rewards/rejected": -2.053563117980957,
      "step": 620
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.352064220183486e-07,
      "logits/chosen": -2.7762598991394043,
      "logits/rejected": -2.8393187522888184,
      "logps/chosen": -231.7938690185547,
      "logps/rejected": -236.4832763671875,
      "loss": 0.514,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.46596893668174744,
      "rewards/margins": 1.896152138710022,
      "rewards/rejected": -2.362121343612671,
      "step": 630
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.3329510703363915e-07,
      "logits/chosen": -3.0272417068481445,
      "logits/rejected": -2.8095943927764893,
      "logps/chosen": -315.4911804199219,
      "logps/rejected": -260.47344970703125,
      "loss": 0.5146,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6075493097305298,
      "rewards/margins": 1.673418402671814,
      "rewards/rejected": -2.2809677124023438,
      "step": 640
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.313837920489297e-07,
      "logits/chosen": -2.8686397075653076,
      "logits/rejected": -2.8369593620300293,
      "logps/chosen": -298.10809326171875,
      "logps/rejected": -342.1612243652344,
      "loss": 0.4628,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6422223448753357,
      "rewards/margins": 1.362321138381958,
      "rewards/rejected": -2.0045435428619385,
      "step": 650
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.2947247706422016e-07,
      "logits/chosen": -2.868638753890991,
      "logits/rejected": -2.918509006500244,
      "logps/chosen": -231.58792114257812,
      "logps/rejected": -275.41357421875,
      "loss": 0.4846,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6644395589828491,
      "rewards/margins": 1.1484898328781128,
      "rewards/rejected": -1.8129297494888306,
      "step": 660
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.275611620795107e-07,
      "logits/chosen": -2.945974349975586,
      "logits/rejected": -2.932396411895752,
      "logps/chosen": -334.83758544921875,
      "logps/rejected": -211.2024688720703,
      "loss": 0.5064,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9673671722412109,
      "rewards/margins": 1.478623628616333,
      "rewards/rejected": -2.445990562438965,
      "step": 670
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.2564984709480123e-07,
      "logits/chosen": -2.999311923980713,
      "logits/rejected": -2.986499309539795,
      "logps/chosen": -356.9630432128906,
      "logps/rejected": -250.017822265625,
      "loss": 0.566,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9130574464797974,
      "rewards/margins": 1.1842575073242188,
      "rewards/rejected": -2.0973148345947266,
      "step": 680
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.2373853211009176e-07,
      "logits/chosen": -2.953401803970337,
      "logits/rejected": -2.949079990386963,
      "logps/chosen": -329.53131103515625,
      "logps/rejected": -244.694091796875,
      "loss": 0.5244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6074694395065308,
      "rewards/margins": 1.2550690174102783,
      "rewards/rejected": -1.8625385761260986,
      "step": 690
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.2182721712538224e-07,
      "logits/chosen": -2.9399592876434326,
      "logits/rejected": -2.97617506980896,
      "logps/chosen": -245.843505859375,
      "logps/rejected": -201.2354736328125,
      "loss": 0.5609,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6896957755088806,
      "rewards/margins": 0.7539043426513672,
      "rewards/rejected": -1.4435999393463135,
      "step": 700
    },
    {
      "epoch": 0.72,
      "eval_logits/chosen": -3.0127830505371094,
      "eval_logits/rejected": -2.947491407394409,
      "eval_logps/chosen": -259.0303649902344,
      "eval_logps/rejected": -213.84410095214844,
      "eval_loss": 0.4927924573421478,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -0.42487117648124695,
      "eval_rewards/margins": 1.2989236116409302,
      "eval_rewards/rejected": -1.7237945795059204,
      "eval_runtime": 42.9706,
      "eval_samples_per_second": 46.543,
      "eval_steps_per_second": 0.372,
      "step": 700
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.199159021406727e-07,
      "logits/chosen": -2.9642043113708496,
      "logits/rejected": -2.843035936355591,
      "logps/chosen": -224.43093872070312,
      "logps/rejected": -250.59646606445312,
      "loss": 0.5174,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9030290842056274,
      "rewards/margins": 0.5707172155380249,
      "rewards/rejected": -1.4737461805343628,
      "step": 710
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.1800458715596325e-07,
      "logits/chosen": -2.856870174407959,
      "logits/rejected": -2.928572654724121,
      "logps/chosen": -238.6852569580078,
      "logps/rejected": -312.410400390625,
      "loss": 0.5465,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6243971586227417,
      "rewards/margins": 1.3136012554168701,
      "rewards/rejected": -1.9379985332489014,
      "step": 720
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.160932721712538e-07,
      "logits/chosen": -2.917513608932495,
      "logits/rejected": -2.7057008743286133,
      "logps/chosen": -279.5296630859375,
      "logps/rejected": -293.60882568359375,
      "loss": 0.4981,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.43406224250793457,
      "rewards/margins": 1.3185964822769165,
      "rewards/rejected": -1.752658486366272,
      "step": 730
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.141819571865443e-07,
      "logits/chosen": -2.76603364944458,
      "logits/rejected": -2.7033543586730957,
      "logps/chosen": -257.63519287109375,
      "logps/rejected": -247.8678436279297,
      "loss": 0.4948,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7212907075881958,
      "rewards/margins": 1.1663872003555298,
      "rewards/rejected": -1.887677788734436,
      "step": 740
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.1227064220183485e-07,
      "logits/chosen": -2.826500654220581,
      "logits/rejected": -2.8519866466522217,
      "logps/chosen": -221.1510009765625,
      "logps/rejected": -242.4302215576172,
      "loss": 0.4638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5560463666915894,
      "rewards/margins": 1.6448535919189453,
      "rewards/rejected": -2.200899839401245,
      "step": 750
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.103593272171253e-07,
      "logits/chosen": -2.749040126800537,
      "logits/rejected": -2.7938055992126465,
      "logps/chosen": -357.89544677734375,
      "logps/rejected": -372.37030029296875,
      "loss": 0.5315,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.965776801109314,
      "rewards/margins": 0.8319292068481445,
      "rewards/rejected": -1.7977060079574585,
      "step": 760
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.0844801223241586e-07,
      "logits/chosen": -2.931018829345703,
      "logits/rejected": -2.884331703186035,
      "logps/chosen": -241.74319458007812,
      "logps/rejected": -216.0192108154297,
      "loss": 0.4671,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7199130058288574,
      "rewards/margins": 0.7224114537239075,
      "rewards/rejected": -1.4423243999481201,
      "step": 770
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.065366972477064e-07,
      "logits/chosen": -2.8741250038146973,
      "logits/rejected": -2.789846658706665,
      "logps/chosen": -280.8083801269531,
      "logps/rejected": -227.48275756835938,
      "loss": 0.4834,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.22569909691810608,
      "rewards/margins": 1.9187443256378174,
      "rewards/rejected": -2.1444432735443115,
      "step": 780
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.046253822629969e-07,
      "logits/chosen": -2.7253146171569824,
      "logits/rejected": -2.700835704803467,
      "logps/chosen": -168.32371520996094,
      "logps/rejected": -203.12583923339844,
      "loss": 0.4728,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5355602502822876,
      "rewards/margins": 0.9603021740913391,
      "rewards/rejected": -1.495862364768982,
      "step": 790
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0271406727828745e-07,
      "logits/chosen": -2.994239091873169,
      "logits/rejected": -2.8887484073638916,
      "logps/chosen": -285.24359130859375,
      "logps/rejected": -230.2656707763672,
      "loss": 0.5338,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.39388689398765564,
      "rewards/margins": 1.406872034072876,
      "rewards/rejected": -1.800758957862854,
      "step": 800
    },
    {
      "epoch": 0.83,
      "eval_logits/chosen": -2.9182941913604736,
      "eval_logits/rejected": -2.8454627990722656,
      "eval_logps/chosen": -256.34844970703125,
      "eval_logps/rejected": -215.71998596191406,
      "eval_loss": 0.4767328202724457,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -0.15668097138404846,
      "eval_rewards/margins": 1.754701018333435,
      "eval_rewards/rejected": -1.9113819599151611,
      "eval_runtime": 42.7022,
      "eval_samples_per_second": 46.836,
      "eval_steps_per_second": 0.375,
      "step": 800
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.00802752293578e-07,
      "logits/chosen": -2.85206937789917,
      "logits/rejected": -2.8010294437408447,
      "logps/chosen": -251.4945068359375,
      "logps/rejected": -256.6982421875,
      "loss": 0.4623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5882171392440796,
      "rewards/margins": 1.6525434255599976,
      "rewards/rejected": -2.240760564804077,
      "step": 810
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.9889143730886847e-07,
      "logits/chosen": -2.8327362537384033,
      "logits/rejected": -2.714212656021118,
      "logps/chosen": -290.8333740234375,
      "logps/rejected": -215.6833953857422,
      "loss": 0.4634,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.06578992307186127,
      "rewards/margins": 2.0252296924591064,
      "rewards/rejected": -2.09101939201355,
      "step": 820
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.96980122324159e-07,
      "logits/chosen": -2.7779641151428223,
      "logits/rejected": -2.794778347015381,
      "logps/chosen": -239.24026489257812,
      "logps/rejected": -245.0064697265625,
      "loss": 0.488,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.22896571457386017,
      "rewards/margins": 1.9688221216201782,
      "rewards/rejected": -2.1977877616882324,
      "step": 830
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.9506880733944953e-07,
      "logits/chosen": -2.826106548309326,
      "logits/rejected": -2.7670514583587646,
      "logps/chosen": -257.3904113769531,
      "logps/rejected": -216.4410858154297,
      "loss": 0.5576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7895628213882446,
      "rewards/margins": 1.1692781448364258,
      "rewards/rejected": -1.9588409662246704,
      "step": 840
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.9315749235474006e-07,
      "logits/chosen": -2.93815016746521,
      "logits/rejected": -2.9787683486938477,
      "logps/chosen": -214.74887084960938,
      "logps/rejected": -249.5219268798828,
      "loss": 0.4896,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9767111539840698,
      "rewards/margins": 0.8558231592178345,
      "rewards/rejected": -1.8325341939926147,
      "step": 850
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.912461773700306e-07,
      "logits/chosen": -2.8489022254943848,
      "logits/rejected": -2.7496302127838135,
      "logps/chosen": -400.4466857910156,
      "logps/rejected": -282.9391174316406,
      "loss": 0.5177,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4085620045661926,
      "rewards/margins": 1.7454227209091187,
      "rewards/rejected": -2.153984785079956,
      "step": 860
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.8933486238532107e-07,
      "logits/chosen": -2.911019802093506,
      "logits/rejected": -2.745748996734619,
      "logps/chosen": -374.48126220703125,
      "logps/rejected": -262.03973388671875,
      "loss": 0.4872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41163554787635803,
      "rewards/margins": 1.7663021087646484,
      "rewards/rejected": -2.1779379844665527,
      "step": 870
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.874235474006116e-07,
      "logits/chosen": -2.762225389480591,
      "logits/rejected": -2.7744991779327393,
      "logps/chosen": -216.38070678710938,
      "logps/rejected": -269.47991943359375,
      "loss": 0.4979,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5625228881835938,
      "rewards/margins": 1.6768239736557007,
      "rewards/rejected": -2.239346981048584,
      "step": 880
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.8551223241590214e-07,
      "logits/chosen": -2.912327766418457,
      "logits/rejected": -2.978773593902588,
      "logps/chosen": -230.28842163085938,
      "logps/rejected": -241.50076293945312,
      "loss": 0.5132,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5826612710952759,
      "rewards/margins": 2.0513291358947754,
      "rewards/rejected": -2.6339900493621826,
      "step": 890
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.8360091743119267e-07,
      "logits/chosen": -2.8714957237243652,
      "logits/rejected": -2.774735689163208,
      "logps/chosen": -286.7504577636719,
      "logps/rejected": -263.4833984375,
      "loss": 0.5039,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.15738129615783691,
      "rewards/margins": 1.1751208305358887,
      "rewards/rejected": -1.3325022459030151,
      "step": 900
    },
    {
      "epoch": 0.93,
      "eval_logits/chosen": -2.909325361251831,
      "eval_logits/rejected": -2.8294677734375,
      "eval_logps/chosen": -255.66744995117188,
      "eval_logps/rejected": -213.5056915283203,
      "eval_loss": 0.48539167642593384,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -0.08858150988817215,
      "eval_rewards/margins": 1.6013730764389038,
      "eval_rewards/rejected": -1.6899546384811401,
      "eval_runtime": 42.9119,
      "eval_samples_per_second": 46.607,
      "eval_steps_per_second": 0.373,
      "step": 900
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.816896024464832e-07,
      "logits/chosen": -2.864403247833252,
      "logits/rejected": -2.858276128768921,
      "logps/chosen": -178.23683166503906,
      "logps/rejected": -204.80947875976562,
      "loss": 0.5099,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1971687376499176,
      "rewards/margins": 1.8712667226791382,
      "rewards/rejected": -2.0684351921081543,
      "step": 910
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.797782874617737e-07,
      "logits/chosen": -2.890958070755005,
      "logits/rejected": -2.8029327392578125,
      "logps/chosen": -315.40106201171875,
      "logps/rejected": -253.34005737304688,
      "loss": 0.4705,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.26785898208618164,
      "rewards/margins": 1.1954081058502197,
      "rewards/rejected": -1.463267207145691,
      "step": 920
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.778669724770642e-07,
      "logits/chosen": -2.864638090133667,
      "logits/rejected": -2.8815560340881348,
      "logps/chosen": -327.3506774902344,
      "logps/rejected": -269.56695556640625,
      "loss": 0.4877,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4065053462982178,
      "rewards/margins": 1.4708530902862549,
      "rewards/rejected": -1.8773584365844727,
      "step": 930
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.7595565749235474e-07,
      "logits/chosen": -2.8528895378112793,
      "logits/rejected": -2.884323835372925,
      "logps/chosen": -277.66949462890625,
      "logps/rejected": -217.2276153564453,
      "loss": 0.4998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3980627954006195,
      "rewards/margins": 1.872488260269165,
      "rewards/rejected": -2.2705509662628174,
      "step": 940
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.740443425076452e-07,
      "logits/chosen": -2.8920092582702637,
      "logits/rejected": -2.8774821758270264,
      "logps/chosen": -274.6985778808594,
      "logps/rejected": -294.6683654785156,
      "loss": 0.4798,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3751869201660156,
      "rewards/margins": 1.2973663806915283,
      "rewards/rejected": -1.6725534200668335,
      "step": 950
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.7213302752293575e-07,
      "logits/chosen": -2.9467930793762207,
      "logits/rejected": -2.8368477821350098,
      "logps/chosen": -275.72119140625,
      "logps/rejected": -213.9237060546875,
      "loss": 0.4694,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.3335360884666443,
      "rewards/margins": 0.5510531663894653,
      "rewards/rejected": -0.8845891952514648,
      "step": 960
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.702217125382263e-07,
      "logits/chosen": -2.8036694526672363,
      "logits/rejected": -2.8077778816223145,
      "logps/chosen": -280.42120361328125,
      "logps/rejected": -295.61383056640625,
      "loss": 0.398,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.2956100106239319,
      "rewards/margins": 1.6340839862823486,
      "rewards/rejected": -1.9296939373016357,
      "step": 970
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.6831039755351677e-07,
      "logits/chosen": -2.758172035217285,
      "logits/rejected": -2.8651859760284424,
      "logps/chosen": -231.38320922851562,
      "logps/rejected": -276.5523986816406,
      "loss": 0.088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8953015208244324,
      "rewards/margins": 4.328494071960449,
      "rewards/rejected": -3.433192729949951,
      "step": 980
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.663990825688073e-07,
      "logits/chosen": -2.7469801902770996,
      "logits/rejected": -2.6729702949523926,
      "logps/chosen": -243.54397583007812,
      "logps/rejected": -240.7235107421875,
      "loss": 0.1085,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.0088516473770142,
      "rewards/margins": 5.048202991485596,
      "rewards/rejected": -4.039350986480713,
      "step": 990
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.6448776758409783e-07,
      "logits/chosen": -2.770702600479126,
      "logits/rejected": -2.7252144813537598,
      "logps/chosen": -246.9398956298828,
      "logps/rejected": -216.59854125976562,
      "loss": 0.0776,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3230278789997101,
      "rewards/margins": 3.6204140186309814,
      "rewards/rejected": -3.2973856925964355,
      "step": 1000
    },
    {
      "epoch": 1.03,
      "eval_logits/chosen": -2.8436782360076904,
      "eval_logits/rejected": -2.757986545562744,
      "eval_logps/chosen": -259.6300048828125,
      "eval_logps/rejected": -221.89271545410156,
      "eval_loss": 0.49383220076560974,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -0.4848347306251526,
      "eval_rewards/margins": 2.0438199043273926,
      "eval_rewards/rejected": -2.5286545753479004,
      "eval_runtime": 43.0225,
      "eval_samples_per_second": 46.487,
      "eval_steps_per_second": 0.372,
      "step": 1000
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.6257645259938836e-07,
      "logits/chosen": -2.832165002822876,
      "logits/rejected": -2.765643835067749,
      "logps/chosen": -281.77423095703125,
      "logps/rejected": -261.037841796875,
      "loss": 0.0777,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.658333957195282,
      "rewards/margins": 5.224586009979248,
      "rewards/rejected": -4.5662522315979,
      "step": 1010
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.606651376146789e-07,
      "logits/chosen": -2.770781993865967,
      "logits/rejected": -2.5953280925750732,
      "logps/chosen": -224.37307739257812,
      "logps/rejected": -231.11038208007812,
      "loss": 0.0863,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5244539380073547,
      "rewards/margins": 4.876093864440918,
      "rewards/rejected": -4.351639747619629,
      "step": 1020
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.5875382262996937e-07,
      "logits/chosen": -2.531419038772583,
      "logits/rejected": -2.5288774967193604,
      "logps/chosen": -202.06593322753906,
      "logps/rejected": -248.90481567382812,
      "loss": 0.0707,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.05567428469657898,
      "rewards/margins": 4.294209957122803,
      "rewards/rejected": -4.349884033203125,
      "step": 1030
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.568425076452599e-07,
      "logits/chosen": -2.777543783187866,
      "logits/rejected": -2.808948040008545,
      "logps/chosen": -216.7978973388672,
      "logps/rejected": -242.88247680664062,
      "loss": 0.0825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09309720993041992,
      "rewards/margins": 4.671463966369629,
      "rewards/rejected": -4.578366279602051,
      "step": 1040
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.5493119266055044e-07,
      "logits/chosen": -2.6939737796783447,
      "logits/rejected": -2.6256613731384277,
      "logps/chosen": -260.4227294921875,
      "logps/rejected": -225.3296356201172,
      "loss": 0.093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5441755056381226,
      "rewards/margins": 5.277462482452393,
      "rewards/rejected": -4.7332868576049805,
      "step": 1050
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.5301987767584097e-07,
      "logits/chosen": -3.0205957889556885,
      "logits/rejected": -2.9231088161468506,
      "logps/chosen": -247.9834442138672,
      "logps/rejected": -340.5820007324219,
      "loss": 0.0714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32120850682258606,
      "rewards/margins": 4.978722095489502,
      "rewards/rejected": -4.657513618469238,
      "step": 1060
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.511085626911315e-07,
      "logits/chosen": -2.9255290031433105,
      "logits/rejected": -2.868518114089966,
      "logps/chosen": -375.0356140136719,
      "logps/rejected": -345.5308837890625,
      "loss": 0.0977,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.7949947118759155,
      "rewards/margins": 6.140542030334473,
      "rewards/rejected": -5.345546722412109,
      "step": 1070
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.49197247706422e-07,
      "logits/chosen": -2.9496195316314697,
      "logits/rejected": -2.848146438598633,
      "logps/chosen": -289.2176208496094,
      "logps/rejected": -232.7750701904297,
      "loss": 0.0992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6688757538795471,
      "rewards/margins": 5.5320024490356445,
      "rewards/rejected": -4.8631272315979,
      "step": 1080
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.472859327217125e-07,
      "logits/chosen": -2.7657835483551025,
      "logits/rejected": -2.740086555480957,
      "logps/chosen": -369.2336120605469,
      "logps/rejected": -343.9190979003906,
      "loss": 0.0875,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.10366223007440567,
      "rewards/margins": 3.979884624481201,
      "rewards/rejected": -3.876222610473633,
      "step": 1090
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.4537461773700304e-07,
      "logits/chosen": -2.8481802940368652,
      "logits/rejected": -2.714881181716919,
      "logps/chosen": -180.77984619140625,
      "logps/rejected": -169.2972412109375,
      "loss": 0.0901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22425577044487,
      "rewards/margins": 4.330163955688477,
      "rewards/rejected": -4.55441951751709,
      "step": 1100
    },
    {
      "epoch": 1.14,
      "eval_logits/chosen": -2.8857600688934326,
      "eval_logits/rejected": -2.803550958633423,
      "eval_logps/chosen": -265.58172607421875,
      "eval_logps/rejected": -229.02471923828125,
      "eval_loss": 0.5071350336074829,
      "eval_rewards/accuracies": 0.78125,
      "eval_rewards/chosen": -1.0800076723098755,
      "eval_rewards/margins": 2.1618502140045166,
      "eval_rewards/rejected": -3.2418575286865234,
      "eval_runtime": 43.4425,
      "eval_samples_per_second": 46.038,
      "eval_steps_per_second": 0.368,
      "step": 1100
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.434633027522936e-07,
      "logits/chosen": -2.6898512840270996,
      "logits/rejected": -2.666327953338623,
      "logps/chosen": -280.2970275878906,
      "logps/rejected": -263.78839111328125,
      "loss": 0.0879,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.7688434720039368,
      "rewards/margins": 5.902011394500732,
      "rewards/rejected": -5.1331682205200195,
      "step": 1110
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.415519877675841e-07,
      "logits/chosen": -2.860400915145874,
      "logits/rejected": -2.8594813346862793,
      "logps/chosen": -216.53561401367188,
      "logps/rejected": -322.01275634765625,
      "loss": 0.085,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3255351185798645,
      "rewards/margins": 4.399062633514404,
      "rewards/rejected": -4.724597930908203,
      "step": 1120
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.3964067278287464e-07,
      "logits/chosen": -2.682889461517334,
      "logits/rejected": -2.781895399093628,
      "logps/chosen": -315.12042236328125,
      "logps/rejected": -330.3970947265625,
      "loss": 0.102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.041519224643707275,
      "rewards/margins": 5.329067230224609,
      "rewards/rejected": -5.370586395263672,
      "step": 1130
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.377293577981651e-07,
      "logits/chosen": -2.815574884414673,
      "logits/rejected": -2.6275439262390137,
      "logps/chosen": -235.42666625976562,
      "logps/rejected": -239.9461669921875,
      "loss": 0.0884,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.18244703114032745,
      "rewards/margins": 4.990113258361816,
      "rewards/rejected": -5.172560691833496,
      "step": 1140
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.3581804281345565e-07,
      "logits/chosen": -3.0224831104278564,
      "logits/rejected": -2.7311410903930664,
      "logps/chosen": -346.58941650390625,
      "logps/rejected": -287.28424072265625,
      "loss": 0.0732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4470561146736145,
      "rewards/margins": 5.634953022003174,
      "rewards/rejected": -5.187896728515625,
      "step": 1150
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.339067278287462e-07,
      "logits/chosen": -2.6187033653259277,
      "logits/rejected": -2.674354076385498,
      "logps/chosen": -232.01919555664062,
      "logps/rejected": -265.0806884765625,
      "loss": 0.0662,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4214044511318207,
      "rewards/margins": 4.962105751037598,
      "rewards/rejected": -4.540700912475586,
      "step": 1160
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.319954128440367e-07,
      "logits/chosen": -2.810262680053711,
      "logits/rejected": -2.6492209434509277,
      "logps/chosen": -365.208251953125,
      "logps/rejected": -247.2949676513672,
      "loss": 0.0673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2072467803955078,
      "rewards/margins": 5.322917938232422,
      "rewards/rejected": -5.530165195465088,
      "step": 1170
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.3008409785932725e-07,
      "logits/chosen": -2.717909097671509,
      "logits/rejected": -2.684145450592041,
      "logps/chosen": -186.22238159179688,
      "logps/rejected": -240.98068237304688,
      "loss": 0.0752,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.022188162431120872,
      "rewards/margins": 5.249354362487793,
      "rewards/rejected": -5.227167129516602,
      "step": 1180
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.2817278287461773e-07,
      "logits/chosen": -2.7536282539367676,
      "logits/rejected": -2.809913158416748,
      "logps/chosen": -212.02871704101562,
      "logps/rejected": -356.77178955078125,
      "loss": 0.0841,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17943891882896423,
      "rewards/margins": 5.2806596755981445,
      "rewards/rejected": -5.460098743438721,
      "step": 1190
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.262614678899082e-07,
      "logits/chosen": -2.661210536956787,
      "logits/rejected": -2.663743495941162,
      "logps/chosen": -237.6588134765625,
      "logps/rejected": -316.5496520996094,
      "loss": 0.0828,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.43801799416542053,
      "rewards/margins": 5.53380012512207,
      "rewards/rejected": -5.9718170166015625,
      "step": 1200
    },
    {
      "epoch": 1.24,
      "eval_logits/chosen": -2.8707656860351562,
      "eval_logits/rejected": -2.7961297035217285,
      "eval_logps/chosen": -264.4635009765625,
      "eval_logps/rejected": -230.69354248046875,
      "eval_loss": 0.5158514380455017,
      "eval_rewards/accuracies": 0.78125,
      "eval_rewards/chosen": -0.9681856036186218,
      "eval_rewards/margins": 2.4405524730682373,
      "eval_rewards/rejected": -3.408737897872925,
      "eval_runtime": 42.8623,
      "eval_samples_per_second": 46.661,
      "eval_steps_per_second": 0.373,
      "step": 1200
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.2435015290519874e-07,
      "logits/chosen": -2.7133708000183105,
      "logits/rejected": -2.6246652603149414,
      "logps/chosen": -197.2749481201172,
      "logps/rejected": -306.0080871582031,
      "loss": 0.0701,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.03982555866241455,
      "rewards/margins": 5.2729597091674805,
      "rewards/rejected": -5.312786102294922,
      "step": 1210
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.2243883792048927e-07,
      "logits/chosen": -2.7416293621063232,
      "logits/rejected": -2.7878434658050537,
      "logps/chosen": -231.7838592529297,
      "logps/rejected": -307.630615234375,
      "loss": 0.0939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4765847623348236,
      "rewards/margins": 6.021305561065674,
      "rewards/rejected": -5.544720649719238,
      "step": 1220
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.205275229357798e-07,
      "logits/chosen": -2.8276381492614746,
      "logits/rejected": -2.740967035293579,
      "logps/chosen": -268.0155334472656,
      "logps/rejected": -276.89300537109375,
      "loss": 0.0942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23615550994873047,
      "rewards/margins": 5.408732891082764,
      "rewards/rejected": -5.172577857971191,
      "step": 1230
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.186162079510703e-07,
      "logits/chosen": -2.8147940635681152,
      "logits/rejected": -2.8905491828918457,
      "logps/chosen": -262.6653747558594,
      "logps/rejected": -325.59783935546875,
      "loss": 0.093,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3194983899593353,
      "rewards/margins": 5.201966285705566,
      "rewards/rejected": -5.521464824676514,
      "step": 1240
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.167048929663608e-07,
      "logits/chosen": -2.7950730323791504,
      "logits/rejected": -2.7414755821228027,
      "logps/chosen": -225.0370635986328,
      "logps/rejected": -268.7679138183594,
      "loss": 0.0925,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6976510286331177,
      "rewards/margins": 4.719433784484863,
      "rewards/rejected": -5.417084693908691,
      "step": 1250
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.1479357798165134e-07,
      "logits/chosen": -2.813150405883789,
      "logits/rejected": -2.7929835319519043,
      "logps/chosen": -238.1572265625,
      "logps/rejected": -248.3897705078125,
      "loss": 0.0929,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.08550971001386642,
      "rewards/margins": 3.972590684890747,
      "rewards/rejected": -4.05810022354126,
      "step": 1260
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.128822629969419e-07,
      "logits/chosen": -2.989551067352295,
      "logits/rejected": -2.859724998474121,
      "logps/chosen": -285.981201171875,
      "logps/rejected": -308.9172058105469,
      "loss": 0.0952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4108548164367676,
      "rewards/margins": 5.075130462646484,
      "rewards/rejected": -5.485985279083252,
      "step": 1270
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.109709480122324e-07,
      "logits/chosen": -2.619586706161499,
      "logits/rejected": -2.7320122718811035,
      "logps/chosen": -277.3897399902344,
      "logps/rejected": -280.6328430175781,
      "loss": 0.0948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4828956127166748,
      "rewards/margins": 7.261415958404541,
      "rewards/rejected": -5.778519630432129,
      "step": 1280
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.0905963302752294e-07,
      "logits/chosen": -2.643578052520752,
      "logits/rejected": -2.671351671218872,
      "logps/chosen": -221.6022186279297,
      "logps/rejected": -271.2128601074219,
      "loss": 0.0954,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.4258726239204407,
      "rewards/margins": 5.396482467651367,
      "rewards/rejected": -5.822355270385742,
      "step": 1290
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.071483180428134e-07,
      "logits/chosen": -2.925110340118408,
      "logits/rejected": -2.775982618331909,
      "logps/chosen": -396.8591003417969,
      "logps/rejected": -287.55902099609375,
      "loss": 0.0916,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8476167917251587,
      "rewards/margins": 5.994976997375488,
      "rewards/rejected": -5.147359371185303,
      "step": 1300
    },
    {
      "epoch": 1.34,
      "eval_logits/chosen": -2.8753504753112793,
      "eval_logits/rejected": -2.801880359649658,
      "eval_logps/chosen": -265.613525390625,
      "eval_logps/rejected": -232.14105224609375,
      "eval_loss": 0.5221606492996216,
      "eval_rewards/accuracies": 0.796875,
      "eval_rewards/chosen": -1.0831860303878784,
      "eval_rewards/margins": 2.4703056812286377,
      "eval_rewards/rejected": -3.5534915924072266,
      "eval_runtime": 42.8737,
      "eval_samples_per_second": 46.649,
      "eval_steps_per_second": 0.373,
      "step": 1300
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0523700305810395e-07,
      "logits/chosen": -2.9040591716766357,
      "logits/rejected": -2.857766628265381,
      "logps/chosen": -282.7038879394531,
      "logps/rejected": -336.998779296875,
      "loss": 0.1023,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.1848091334104538,
      "rewards/margins": 5.379525661468506,
      "rewards/rejected": -5.194716453552246,
      "step": 1310
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.033256880733945e-07,
      "logits/chosen": -2.662896156311035,
      "logits/rejected": -2.662188768386841,
      "logps/chosen": -175.68914794921875,
      "logps/rejected": -254.7020263671875,
      "loss": 0.0944,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9065677523612976,
      "rewards/margins": 3.604799270629883,
      "rewards/rejected": -4.511366844177246,
      "step": 1320
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.01414373088685e-07,
      "logits/chosen": -2.8199753761291504,
      "logits/rejected": -2.8096752166748047,
      "logps/chosen": -276.9725341796875,
      "logps/rejected": -313.1163330078125,
      "loss": 0.1005,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.162560373544693,
      "rewards/margins": 5.922631740570068,
      "rewards/rejected": -5.760071277618408,
      "step": 1330
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.9950305810397555e-07,
      "logits/chosen": -2.7838358879089355,
      "logits/rejected": -2.910243511199951,
      "logps/chosen": -271.54608154296875,
      "logps/rejected": -326.84429931640625,
      "loss": 0.0981,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6067371964454651,
      "rewards/margins": 6.717789649963379,
      "rewards/rejected": -7.324526309967041,
      "step": 1340
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9759174311926603e-07,
      "logits/chosen": -2.840893507003784,
      "logits/rejected": -2.637371063232422,
      "logps/chosen": -319.387451171875,
      "logps/rejected": -196.8367462158203,
      "loss": 0.0884,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.37663254141807556,
      "rewards/margins": 4.401788234710693,
      "rewards/rejected": -4.778420448303223,
      "step": 1350
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9568042813455656e-07,
      "logits/chosen": -2.739501714706421,
      "logits/rejected": -2.7261953353881836,
      "logps/chosen": -247.82839965820312,
      "logps/rejected": -260.4093017578125,
      "loss": 0.1072,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5568360090255737,
      "rewards/margins": 4.246486663818359,
      "rewards/rejected": -4.803322792053223,
      "step": 1360
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.937691131498471e-07,
      "logits/chosen": -2.8332924842834473,
      "logits/rejected": -2.639050245285034,
      "logps/chosen": -309.36627197265625,
      "logps/rejected": -355.98858642578125,
      "loss": 0.0911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0330581665039062,
      "rewards/margins": 7.80413818359375,
      "rewards/rejected": -6.771079063415527,
      "step": 1370
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.918577981651376e-07,
      "logits/chosen": -2.7822353839874268,
      "logits/rejected": -2.569251537322998,
      "logps/chosen": -317.1792297363281,
      "logps/rejected": -229.3711395263672,
      "loss": 0.0942,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5411940813064575,
      "rewards/margins": 6.266817569732666,
      "rewards/rejected": -5.72562313079834,
      "step": 1380
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.8994648318042816e-07,
      "logits/chosen": -2.987076759338379,
      "logits/rejected": -2.8996262550354004,
      "logps/chosen": -253.8076629638672,
      "logps/rejected": -246.89614868164062,
      "loss": 0.0875,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.017121601849794388,
      "rewards/margins": 4.667881965637207,
      "rewards/rejected": -4.68500280380249,
      "step": 1390
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8803516819571863e-07,
      "logits/chosen": -2.9189207553863525,
      "logits/rejected": -2.7888290882110596,
      "logps/chosen": -307.88287353515625,
      "logps/rejected": -285.6153564453125,
      "loss": 0.0965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3171554207801819,
      "rewards/margins": 6.538872718811035,
      "rewards/rejected": -6.22171688079834,
      "step": 1400
    },
    {
      "epoch": 1.44,
      "eval_logits/chosen": -2.888350009918213,
      "eval_logits/rejected": -2.8058454990386963,
      "eval_logps/chosen": -266.73236083984375,
      "eval_logps/rejected": -232.28741455078125,
      "eval_loss": 0.5203580260276794,
      "eval_rewards/accuracies": 0.796875,
      "eval_rewards/chosen": -1.1950702667236328,
      "eval_rewards/margins": 2.373054027557373,
      "eval_rewards/rejected": -3.568124532699585,
      "eval_runtime": 43.2124,
      "eval_samples_per_second": 46.283,
      "eval_steps_per_second": 0.37,
      "step": 1400
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.8612385321100917e-07,
      "logits/chosen": -2.74180269241333,
      "logits/rejected": -2.743090867996216,
      "logps/chosen": -199.86988830566406,
      "logps/rejected": -292.73370361328125,
      "loss": 0.0872,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6563189625740051,
      "rewards/margins": 6.341017246246338,
      "rewards/rejected": -5.684699058532715,
      "step": 1410
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.842125382262997e-07,
      "logits/chosen": -2.712198495864868,
      "logits/rejected": -2.7786831855773926,
      "logps/chosen": -284.89263916015625,
      "logps/rejected": -333.57489013671875,
      "loss": 0.0812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07811599969863892,
      "rewards/margins": 6.2695770263671875,
      "rewards/rejected": -6.191461563110352,
      "step": 1420
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.8230122324159023e-07,
      "logits/chosen": -2.4872963428497314,
      "logits/rejected": -2.634469747543335,
      "logps/chosen": -216.0610809326172,
      "logps/rejected": -366.69879150390625,
      "loss": 0.0868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7169110178947449,
      "rewards/margins": 6.171887397766113,
      "rewards/rejected": -6.888798713684082,
      "step": 1430
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.8038990825688076e-07,
      "logits/chosen": -2.914641857147217,
      "logits/rejected": -2.7360188961029053,
      "logps/chosen": -219.3689727783203,
      "logps/rejected": -247.0272216796875,
      "loss": 0.0954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9662607312202454,
      "rewards/margins": 4.363907337188721,
      "rewards/rejected": -5.330168724060059,
      "step": 1440
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.784785932721712e-07,
      "logits/chosen": -2.8227076530456543,
      "logits/rejected": -2.789245367050171,
      "logps/chosen": -270.40057373046875,
      "logps/rejected": -279.9286804199219,
      "loss": 0.1157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15661029517650604,
      "rewards/margins": 5.467320442199707,
      "rewards/rejected": -5.623930931091309,
      "step": 1450
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.765672782874617e-07,
      "logits/chosen": -2.7216007709503174,
      "logits/rejected": -2.6529860496520996,
      "logps/chosen": -319.59832763671875,
      "logps/rejected": -283.9094543457031,
      "loss": 0.0845,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.5374232530593872,
      "rewards/margins": 7.195462226867676,
      "rewards/rejected": -5.65803861618042,
      "step": 1460
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.7465596330275225e-07,
      "logits/chosen": -2.7975354194641113,
      "logits/rejected": -2.7123022079467773,
      "logps/chosen": -226.4639129638672,
      "logps/rejected": -204.92037963867188,
      "loss": 0.087,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1759016215801239,
      "rewards/margins": 4.392673015594482,
      "rewards/rejected": -4.568574905395508,
      "step": 1470
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.727446483180428e-07,
      "logits/chosen": -2.9199295043945312,
      "logits/rejected": -2.9144251346588135,
      "logps/chosen": -325.1085205078125,
      "logps/rejected": -300.2225341796875,
      "loss": 0.0771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7285654544830322,
      "rewards/margins": 5.159348011016846,
      "rewards/rejected": -5.887913227081299,
      "step": 1480
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.708333333333333e-07,
      "logits/chosen": -2.852546453475952,
      "logits/rejected": -2.7731895446777344,
      "logps/chosen": -293.71307373046875,
      "logps/rejected": -298.82818603515625,
      "loss": 0.0781,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.1010526418685913,
      "rewards/margins": 5.532158374786377,
      "rewards/rejected": -6.633211612701416,
      "step": 1490
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.6892201834862385e-07,
      "logits/chosen": -2.972316265106201,
      "logits/rejected": -2.7817482948303223,
      "logps/chosen": -322.0675354003906,
      "logps/rejected": -286.2783203125,
      "loss": 0.0716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5001612901687622,
      "rewards/margins": 6.432877540588379,
      "rewards/rejected": -5.9327168464660645,
      "step": 1500
    },
    {
      "epoch": 1.55,
      "eval_logits/chosen": -2.886213779449463,
      "eval_logits/rejected": -2.797916889190674,
      "eval_logps/chosen": -271.3697204589844,
      "eval_logps/rejected": -237.4441375732422,
      "eval_loss": 0.538065493106842,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -1.6588083505630493,
      "eval_rewards/margins": 2.424990177154541,
      "eval_rewards/rejected": -4.083798408508301,
      "eval_runtime": 42.6303,
      "eval_samples_per_second": 46.915,
      "eval_steps_per_second": 0.375,
      "step": 1500
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.6701070336391433e-07,
      "logits/chosen": -2.6193156242370605,
      "logits/rejected": -2.575343608856201,
      "logps/chosen": -286.7375793457031,
      "logps/rejected": -227.78652954101562,
      "loss": 0.0781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.46129971742630005,
      "rewards/margins": 5.609195232391357,
      "rewards/rejected": -5.147895812988281,
      "step": 1510
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.6509938837920486e-07,
      "logits/chosen": -2.908501148223877,
      "logits/rejected": -2.6500637531280518,
      "logps/chosen": -303.86737060546875,
      "logps/rejected": -269.2611999511719,
      "loss": 0.0936,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5135625600814819,
      "rewards/margins": 5.414787292480469,
      "rewards/rejected": -4.901224613189697,
      "step": 1520
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.631880733944954e-07,
      "logits/chosen": -2.8338589668273926,
      "logits/rejected": -2.70253586769104,
      "logps/chosen": -328.9350891113281,
      "logps/rejected": -259.4769287109375,
      "loss": 0.0785,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.37550514936447144,
      "rewards/margins": 4.032449245452881,
      "rewards/rejected": -4.407954692840576,
      "step": 1530
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.612767584097859e-07,
      "logits/chosen": -2.8646657466888428,
      "logits/rejected": -2.771764039993286,
      "logps/chosen": -266.7498779296875,
      "logps/rejected": -286.897705078125,
      "loss": 0.1001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10361538082361221,
      "rewards/margins": 5.30898380279541,
      "rewards/rejected": -5.205368995666504,
      "step": 1540
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.5936544342507646e-07,
      "logits/chosen": -2.8486104011535645,
      "logits/rejected": -2.7594268321990967,
      "logps/chosen": -289.22979736328125,
      "logps/rejected": -331.20989990234375,
      "loss": 0.1022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9251961708068848,
      "rewards/margins": 7.291400909423828,
      "rewards/rejected": -6.366204261779785,
      "step": 1550
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.5745412844036693e-07,
      "logits/chosen": -2.771709442138672,
      "logits/rejected": -2.580551862716675,
      "logps/chosen": -304.80816650390625,
      "logps/rejected": -289.4775085449219,
      "loss": 0.0944,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09966902434825897,
      "rewards/margins": 4.944860458374023,
      "rewards/rejected": -5.044528961181641,
      "step": 1560
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.5554281345565747e-07,
      "logits/chosen": -2.9294638633728027,
      "logits/rejected": -2.6105856895446777,
      "logps/chosen": -351.17864990234375,
      "logps/rejected": -303.7239685058594,
      "loss": 0.0834,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7888059616088867,
      "rewards/margins": 5.206669807434082,
      "rewards/rejected": -5.995476722717285,
      "step": 1570
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.53631498470948e-07,
      "logits/chosen": -2.8216099739074707,
      "logits/rejected": -2.7941040992736816,
      "logps/chosen": -278.52447509765625,
      "logps/rejected": -272.856689453125,
      "loss": 0.0802,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4856332242488861,
      "rewards/margins": 6.397647857666016,
      "rewards/rejected": -5.912014961242676,
      "step": 1580
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.5172018348623853e-07,
      "logits/chosen": -2.9010136127471924,
      "logits/rejected": -2.6620776653289795,
      "logps/chosen": -338.17889404296875,
      "logps/rejected": -222.8809814453125,
      "loss": 0.0808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6562628746032715,
      "rewards/margins": 5.940356254577637,
      "rewards/rejected": -5.284094333648682,
      "step": 1590
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.4980886850152906e-07,
      "logits/chosen": -2.8568649291992188,
      "logits/rejected": -2.7494237422943115,
      "logps/chosen": -379.17425537109375,
      "logps/rejected": -371.10675048828125,
      "loss": 0.0957,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17441731691360474,
      "rewards/margins": 6.062676429748535,
      "rewards/rejected": -5.888258934020996,
      "step": 1600
    },
    {
      "epoch": 1.65,
      "eval_logits/chosen": -2.897590398788452,
      "eval_logits/rejected": -2.7959723472595215,
      "eval_logps/chosen": -266.52783203125,
      "eval_logps/rejected": -234.0833740234375,
      "eval_loss": 0.5150620341300964,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -1.174619197845459,
      "eval_rewards/margins": 2.5731022357940674,
      "eval_rewards/rejected": -3.7477216720581055,
      "eval_runtime": 42.682,
      "eval_samples_per_second": 46.858,
      "eval_steps_per_second": 0.375,
      "step": 1600
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.478975535168196e-07,
      "logits/chosen": -2.6691880226135254,
      "logits/rejected": -2.6619346141815186,
      "logps/chosen": -242.07443237304688,
      "logps/rejected": -234.34457397460938,
      "loss": 0.0824,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.0017588138580322266,
      "rewards/margins": 4.478671550750732,
      "rewards/rejected": -4.480430603027344,
      "step": 1610
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.459862385321101e-07,
      "logits/chosen": -2.891451835632324,
      "logits/rejected": -2.8165245056152344,
      "logps/chosen": -386.6529235839844,
      "logps/rejected": -339.13525390625,
      "loss": 0.0896,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5312672257423401,
      "rewards/margins": 6.819835662841797,
      "rewards/rejected": -6.288567543029785,
      "step": 1620
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.440749235474006e-07,
      "logits/chosen": -2.6342310905456543,
      "logits/rejected": -2.497105836868286,
      "logps/chosen": -298.6165771484375,
      "logps/rejected": -333.8529968261719,
      "loss": 0.0812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11463097482919693,
      "rewards/margins": 6.6383843421936035,
      "rewards/rejected": -6.523752689361572,
      "step": 1630
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.421636085626911e-07,
      "logits/chosen": -2.8128368854522705,
      "logits/rejected": -2.6747617721557617,
      "logps/chosen": -333.1402282714844,
      "logps/rejected": -336.7758483886719,
      "loss": 0.072,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.7955392003059387,
      "rewards/margins": 6.469751834869385,
      "rewards/rejected": -5.674212455749512,
      "step": 1640
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.402522935779816e-07,
      "logits/chosen": -2.743809223175049,
      "logits/rejected": -2.582720994949341,
      "logps/chosen": -312.7721862792969,
      "logps/rejected": -325.0949401855469,
      "loss": 0.0771,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18811270594596863,
      "rewards/margins": 7.560137748718262,
      "rewards/rejected": -7.372025489807129,
      "step": 1650
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.3834097859327215e-07,
      "logits/chosen": -2.766622304916382,
      "logits/rejected": -2.711757183074951,
      "logps/chosen": -232.3721466064453,
      "logps/rejected": -341.15313720703125,
      "loss": 0.0908,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.0886591449379921,
      "rewards/margins": 7.887714385986328,
      "rewards/rejected": -7.976373195648193,
      "step": 1660
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.3642966360856268e-07,
      "logits/chosen": -2.8221402168273926,
      "logits/rejected": -2.8537559509277344,
      "logps/chosen": -290.0625,
      "logps/rejected": -268.5934753417969,
      "loss": 0.0952,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.37126991152763367,
      "rewards/margins": 4.8282670974731445,
      "rewards/rejected": -5.199536323547363,
      "step": 1670
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.345183486238532e-07,
      "logits/chosen": -2.6492087841033936,
      "logits/rejected": -2.7734198570251465,
      "logps/chosen": -258.0906066894531,
      "logps/rejected": -298.752197265625,
      "loss": 0.0758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40164145827293396,
      "rewards/margins": 5.819032192230225,
      "rewards/rejected": -5.4173903465271,
      "step": 1680
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.3260703363914372e-07,
      "logits/chosen": -2.8272266387939453,
      "logits/rejected": -2.8661937713623047,
      "logps/chosen": -342.8156433105469,
      "logps/rejected": -301.5121765136719,
      "loss": 0.0946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8689159154891968,
      "rewards/margins": 5.930572032928467,
      "rewards/rejected": -5.0616559982299805,
      "step": 1690
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.3069571865443425e-07,
      "logits/chosen": -2.723177671432495,
      "logits/rejected": -2.4498984813690186,
      "logps/chosen": -235.50997924804688,
      "logps/rejected": -218.8724365234375,
      "loss": 0.0645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9274832010269165,
      "rewards/margins": 4.0410966873168945,
      "rewards/rejected": -4.96858024597168,
      "step": 1700
    },
    {
      "epoch": 1.75,
      "eval_logits/chosen": -2.8591511249542236,
      "eval_logits/rejected": -2.748270034790039,
      "eval_logps/chosen": -272.372802734375,
      "eval_logps/rejected": -242.61672973632812,
      "eval_loss": 0.5392942428588867,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -1.7591183185577393,
      "eval_rewards/margins": 2.841939926147461,
      "eval_rewards/rejected": -4.601057529449463,
      "eval_runtime": 43.4291,
      "eval_samples_per_second": 46.052,
      "eval_steps_per_second": 0.368,
      "step": 1700
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.2878440366972476e-07,
      "logits/chosen": -2.917429208755493,
      "logits/rejected": -2.831324338912964,
      "logps/chosen": -285.4388122558594,
      "logps/rejected": -409.280517578125,
      "loss": 0.0811,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.4222320318222046,
      "rewards/margins": 4.949131488800049,
      "rewards/rejected": -6.371363162994385,
      "step": 1710
    },
    {
      "epoch": 1.78,
      "learning_rate": 2.268730886850153e-07,
      "logits/chosen": -2.5324103832244873,
      "logits/rejected": -2.5078229904174805,
      "logps/chosen": -213.4552459716797,
      "logps/rejected": -233.57766723632812,
      "loss": 0.0777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24673457443714142,
      "rewards/margins": 6.043379783630371,
      "rewards/rejected": -5.796644687652588,
      "step": 1720
    },
    {
      "epoch": 1.79,
      "learning_rate": 2.249617737003058e-07,
      "logits/chosen": -2.5907716751098633,
      "logits/rejected": -2.6474852561950684,
      "logps/chosen": -296.5213317871094,
      "logps/rejected": -345.16925048828125,
      "loss": 0.094,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.4688749313354492,
      "rewards/margins": 7.06952428817749,
      "rewards/rejected": -6.600649833679199,
      "step": 1730
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.2305045871559633e-07,
      "logits/chosen": -2.7180168628692627,
      "logits/rejected": -2.6339097023010254,
      "logps/chosen": -273.3642578125,
      "logps/rejected": -363.3600769042969,
      "loss": 0.0755,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16731056571006775,
      "rewards/margins": 6.712351322174072,
      "rewards/rejected": -6.545041084289551,
      "step": 1740
    },
    {
      "epoch": 1.81,
      "learning_rate": 2.2113914373088686e-07,
      "logits/chosen": -2.711622953414917,
      "logits/rejected": -2.8453264236450195,
      "logps/chosen": -166.16708374023438,
      "logps/rejected": -307.5210266113281,
      "loss": 0.0886,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6864116191864014,
      "rewards/margins": 5.068359375,
      "rewards/rejected": -5.754770755767822,
      "step": 1750
    },
    {
      "epoch": 1.82,
      "learning_rate": 2.1922782874617736e-07,
      "logits/chosen": -2.812593460083008,
      "logits/rejected": -2.9017271995544434,
      "logps/chosen": -255.8525848388672,
      "logps/rejected": -350.1146240234375,
      "loss": 0.0824,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.615868091583252,
      "rewards/margins": 7.678023338317871,
      "rewards/rejected": -7.062155246734619,
      "step": 1760
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.1731651376146787e-07,
      "logits/chosen": -2.7244350910186768,
      "logits/rejected": -2.595527172088623,
      "logps/chosen": -258.34674072265625,
      "logps/rejected": -282.9205627441406,
      "loss": 0.0807,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06415136158466339,
      "rewards/margins": 4.976747035980225,
      "rewards/rejected": -4.912595272064209,
      "step": 1770
    },
    {
      "epoch": 1.84,
      "learning_rate": 2.154051987767584e-07,
      "logits/chosen": -2.7545602321624756,
      "logits/rejected": -2.7094874382019043,
      "logps/chosen": -297.81500244140625,
      "logps/rejected": -347.14471435546875,
      "loss": 0.0747,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3948817849159241,
      "rewards/margins": 4.995589733123779,
      "rewards/rejected": -5.390471935272217,
      "step": 1780
    },
    {
      "epoch": 1.85,
      "learning_rate": 2.134938837920489e-07,
      "logits/chosen": -2.6277823448181152,
      "logits/rejected": -2.582498550415039,
      "logps/chosen": -216.9835662841797,
      "logps/rejected": -271.9996337890625,
      "loss": 0.0651,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8243429064750671,
      "rewards/margins": 5.609129905700684,
      "rewards/rejected": -6.433472633361816,
      "step": 1790
    },
    {
      "epoch": 1.86,
      "learning_rate": 2.1158256880733944e-07,
      "logits/chosen": -2.853558301925659,
      "logits/rejected": -2.6372504234313965,
      "logps/chosen": -347.44024658203125,
      "logps/rejected": -251.0436553955078,
      "loss": 0.0838,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.08665396273136139,
      "rewards/margins": 6.390503406524658,
      "rewards/rejected": -6.303849697113037,
      "step": 1800
    },
    {
      "epoch": 1.86,
      "eval_logits/chosen": -2.8382647037506104,
      "eval_logits/rejected": -2.7311384677886963,
      "eval_logps/chosen": -271.38751220703125,
      "eval_logps/rejected": -241.25450134277344,
      "eval_loss": 0.5384629964828491,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -1.6605874300003052,
      "eval_rewards/margins": 2.804246425628662,
      "eval_rewards/rejected": -4.464834213256836,
      "eval_runtime": 42.7598,
      "eval_samples_per_second": 46.773,
      "eval_steps_per_second": 0.374,
      "step": 1800
    },
    {
      "epoch": 1.87,
      "learning_rate": 2.0967125382262994e-07,
      "logits/chosen": -2.921271324157715,
      "logits/rejected": -2.6984505653381348,
      "logps/chosen": -233.25341796875,
      "logps/rejected": -301.328125,
      "loss": 0.0846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09845151007175446,
      "rewards/margins": 6.211066246032715,
      "rewards/rejected": -6.112614631652832,
      "step": 1810
    },
    {
      "epoch": 1.88,
      "learning_rate": 2.0775993883792048e-07,
      "logits/chosen": -2.722374677658081,
      "logits/rejected": -2.577514410018921,
      "logps/chosen": -351.37493896484375,
      "logps/rejected": -284.2919616699219,
      "loss": 0.0788,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9589091539382935,
      "rewards/margins": 5.428890228271484,
      "rewards/rejected": -6.3877997398376465,
      "step": 1820
    },
    {
      "epoch": 1.89,
      "learning_rate": 2.05848623853211e-07,
      "logits/chosen": -2.9829728603363037,
      "logits/rejected": -2.8239240646362305,
      "logps/chosen": -316.7395935058594,
      "logps/rejected": -299.128173828125,
      "loss": 0.0881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36243343353271484,
      "rewards/margins": 5.3592963218688965,
      "rewards/rejected": -5.7217302322387695,
      "step": 1830
    },
    {
      "epoch": 1.9,
      "learning_rate": 2.0393730886850151e-07,
      "logits/chosen": -2.8860998153686523,
      "logits/rejected": -2.7891647815704346,
      "logps/chosen": -238.3377227783203,
      "logps/rejected": -254.6820831298828,
      "loss": 0.0682,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6590354442596436,
      "rewards/margins": 4.998110294342041,
      "rewards/rejected": -5.6571455001831055,
      "step": 1840
    },
    {
      "epoch": 1.91,
      "learning_rate": 2.0202599388379205e-07,
      "logits/chosen": -2.9648146629333496,
      "logits/rejected": -2.9021639823913574,
      "logps/chosen": -235.9986114501953,
      "logps/rejected": -505.4097595214844,
      "loss": 0.0892,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.47126349806785583,
      "rewards/margins": 7.691262245178223,
      "rewards/rejected": -8.16252613067627,
      "step": 1850
    },
    {
      "epoch": 1.92,
      "learning_rate": 2.0011467889908258e-07,
      "logits/chosen": -2.822788953781128,
      "logits/rejected": -2.631824254989624,
      "logps/chosen": -326.16778564453125,
      "logps/rejected": -283.84796142578125,
      "loss": 0.0818,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3089871108531952,
      "rewards/margins": 6.355311393737793,
      "rewards/rejected": -6.664299011230469,
      "step": 1860
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.9820336391437308e-07,
      "logits/chosen": -2.7996997833251953,
      "logits/rejected": -2.878431797027588,
      "logps/chosen": -291.4371337890625,
      "logps/rejected": -283.9778747558594,
      "loss": 0.0801,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3241414725780487,
      "rewards/margins": 5.393677711486816,
      "rewards/rejected": -5.7178192138671875,
      "step": 1870
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.9629204892966362e-07,
      "logits/chosen": -2.98240327835083,
      "logits/rejected": -2.914886951446533,
      "logps/chosen": -396.53271484375,
      "logps/rejected": -338.84820556640625,
      "loss": 0.0744,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.0023688077926635742,
      "rewards/margins": 6.182188510894775,
      "rewards/rejected": -6.184557914733887,
      "step": 1880
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.943807339449541e-07,
      "logits/chosen": -2.5186867713928223,
      "logits/rejected": -2.5948076248168945,
      "logps/chosen": -219.8833465576172,
      "logps/rejected": -241.77197265625,
      "loss": 0.0769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.869981586933136,
      "rewards/margins": 5.099352836608887,
      "rewards/rejected": -5.969334602355957,
      "step": 1890
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.9246941896024463e-07,
      "logits/chosen": -2.75243878364563,
      "logits/rejected": -2.6535823345184326,
      "logps/chosen": -186.34742736816406,
      "logps/rejected": -197.3961181640625,
      "loss": 0.1106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7161014676094055,
      "rewards/margins": 6.270589351654053,
      "rewards/rejected": -6.986691951751709,
      "step": 1900
    },
    {
      "epoch": 1.96,
      "eval_logits/chosen": -2.9132931232452393,
      "eval_logits/rejected": -2.8193860054016113,
      "eval_logps/chosen": -270.4024963378906,
      "eval_logps/rejected": -236.38502502441406,
      "eval_loss": 0.5321760177612305,
      "eval_rewards/accuracies": 0.796875,
      "eval_rewards/chosen": -1.5620850324630737,
      "eval_rewards/margins": 2.4158010482788086,
      "eval_rewards/rejected": -3.977886199951172,
      "eval_runtime": 43.3233,
      "eval_samples_per_second": 46.164,
      "eval_steps_per_second": 0.369,
      "step": 1900
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.9055810397553516e-07,
      "logits/chosen": -2.850160598754883,
      "logits/rejected": -2.7610700130462646,
      "logps/chosen": -312.760498046875,
      "logps/rejected": -301.4549255371094,
      "loss": 0.0906,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.179014191031456,
      "rewards/margins": 6.159252166748047,
      "rewards/rejected": -6.338266849517822,
      "step": 1910
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.8864678899082566e-07,
      "logits/chosen": -2.8218185901641846,
      "logits/rejected": -2.8268847465515137,
      "logps/chosen": -292.2250061035156,
      "logps/rejected": -259.8616027832031,
      "loss": 0.0837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9055793881416321,
      "rewards/margins": 4.949950218200684,
      "rewards/rejected": -5.85552978515625,
      "step": 1920
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.867354740061162e-07,
      "logits/chosen": -2.836451530456543,
      "logits/rejected": -2.727323055267334,
      "logps/chosen": -257.83428955078125,
      "logps/rejected": -360.0964050292969,
      "loss": 0.0977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.067908525466919,
      "rewards/margins": 5.707231044769287,
      "rewards/rejected": -6.775138854980469,
      "step": 1930
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.8482415902140673e-07,
      "logits/chosen": -2.6650755405426025,
      "logits/rejected": -2.539886474609375,
      "logps/chosen": -234.5982208251953,
      "logps/rejected": -312.34375,
      "loss": 0.0677,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1976377964019775,
      "rewards/margins": 5.782257556915283,
      "rewards/rejected": -6.979895114898682,
      "step": 1940
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.8291284403669723e-07,
      "logits/chosen": -2.880758285522461,
      "logits/rejected": -2.8343312740325928,
      "logps/chosen": -205.7776336669922,
      "logps/rejected": -282.3731994628906,
      "loss": 0.0222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2522627115249634,
      "rewards/margins": 5.962088584899902,
      "rewards/rejected": -5.709825038909912,
      "step": 1950
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.8100152905198777e-07,
      "logits/chosen": -2.331038475036621,
      "logits/rejected": -2.5355992317199707,
      "logps/chosen": -274.9292907714844,
      "logps/rejected": -410.0576171875,
      "loss": 0.0145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.284432590007782,
      "rewards/margins": 8.246986389160156,
      "rewards/rejected": -8.53141975402832,
      "step": 1960
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.7909021406727827e-07,
      "logits/chosen": -2.6835944652557373,
      "logits/rejected": -2.727116107940674,
      "logps/chosen": -190.50384521484375,
      "logps/rejected": -258.0693664550781,
      "loss": 0.0214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4610201120376587,
      "rewards/margins": 6.77774715423584,
      "rewards/rejected": -8.238768577575684,
      "step": 1970
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.771788990825688e-07,
      "logits/chosen": -2.8552534580230713,
      "logits/rejected": -2.79771089553833,
      "logps/chosen": -312.20550537109375,
      "logps/rejected": -283.9766540527344,
      "loss": 0.0136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4660851061344147,
      "rewards/margins": 7.324423313140869,
      "rewards/rejected": -7.7905073165893555,
      "step": 1980
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.7526758409785934e-07,
      "logits/chosen": -2.8241209983825684,
      "logits/rejected": -2.612579584121704,
      "logps/chosen": -366.5120849609375,
      "logps/rejected": -359.84820556640625,
      "loss": 0.0105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40640372037887573,
      "rewards/margins": 8.778280258178711,
      "rewards/rejected": -9.184685707092285,
      "step": 1990
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.7335626911314984e-07,
      "logits/chosen": -2.6581811904907227,
      "logits/rejected": -2.6591739654541016,
      "logps/chosen": -254.0058135986328,
      "logps/rejected": -281.71539306640625,
      "loss": 0.0174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3120492696762085,
      "rewards/margins": 7.992934226989746,
      "rewards/rejected": -9.304984092712402,
      "step": 2000
    },
    {
      "epoch": 2.06,
      "eval_logits/chosen": -2.863063097000122,
      "eval_logits/rejected": -2.7579469680786133,
      "eval_logps/chosen": -279.7497863769531,
      "eval_logps/rejected": -256.11993408203125,
      "eval_loss": 0.592135488986969,
      "eval_rewards/accuracies": 0.796875,
      "eval_rewards/chosen": -2.496814250946045,
      "eval_rewards/margins": 3.454561948776245,
      "eval_rewards/rejected": -5.951375961303711,
      "eval_runtime": 42.8121,
      "eval_samples_per_second": 46.716,
      "eval_steps_per_second": 0.374,
      "step": 2000
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.7144495412844037e-07,
      "logits/chosen": -3.115104913711548,
      "logits/rejected": -2.79913330078125,
      "logps/chosen": -401.72235107421875,
      "logps/rejected": -373.38885498046875,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9059139490127563,
      "rewards/margins": 8.271968841552734,
      "rewards/rejected": -9.177882194519043,
      "step": 2010
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.6953363914373088e-07,
      "logits/chosen": -2.8683724403381348,
      "logits/rejected": -2.759824752807617,
      "logps/chosen": -328.4763488769531,
      "logps/rejected": -324.87188720703125,
      "loss": 0.0096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38512125611305237,
      "rewards/margins": 10.028793334960938,
      "rewards/rejected": -9.643671989440918,
      "step": 2020
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.6762232415902138e-07,
      "logits/chosen": -2.7730555534362793,
      "logits/rejected": -2.7145891189575195,
      "logps/chosen": -264.50897216796875,
      "logps/rejected": -298.73931884765625,
      "loss": 0.0177,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.688241958618164,
      "rewards/margins": 6.888033390045166,
      "rewards/rejected": -9.576274871826172,
      "step": 2030
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.6571100917431192e-07,
      "logits/chosen": -2.8545258045196533,
      "logits/rejected": -2.622382640838623,
      "logps/chosen": -253.060791015625,
      "logps/rejected": -260.3923645019531,
      "loss": 0.0133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4856998026371002,
      "rewards/margins": 8.40421199798584,
      "rewards/rejected": -8.889912605285645,
      "step": 2040
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.6379969418960242e-07,
      "logits/chosen": -2.813387870788574,
      "logits/rejected": -2.8429861068725586,
      "logps/chosen": -388.98516845703125,
      "logps/rejected": -383.153076171875,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13710257411003113,
      "rewards/margins": 8.103338241577148,
      "rewards/rejected": -8.240442276000977,
      "step": 2050
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.6188837920489295e-07,
      "logits/chosen": -2.682894706726074,
      "logits/rejected": -2.447547197341919,
      "logps/chosen": -270.86126708984375,
      "logps/rejected": -295.6491394042969,
      "loss": 0.0117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3708886504173279,
      "rewards/margins": 8.017080307006836,
      "rewards/rejected": -8.387969970703125,
      "step": 2060
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.5997706422018349e-07,
      "logits/chosen": -2.909653663635254,
      "logits/rejected": -2.926835536956787,
      "logps/chosen": -334.53753662109375,
      "logps/rejected": -379.61907958984375,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7136735916137695,
      "rewards/margins": 9.404067993164062,
      "rewards/rejected": -7.690394401550293,
      "step": 2070
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.58065749235474e-07,
      "logits/chosen": -2.819035768508911,
      "logits/rejected": -2.7074294090270996,
      "logps/chosen": -294.74957275390625,
      "logps/rejected": -300.90716552734375,
      "loss": 0.0124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25709009170532227,
      "rewards/margins": 7.958430290222168,
      "rewards/rejected": -8.215520858764648,
      "step": 2080
    },
    {
      "epoch": 2.16,
      "learning_rate": 1.5615443425076452e-07,
      "logits/chosen": -2.8023552894592285,
      "logits/rejected": -2.638780355453491,
      "logps/chosen": -327.9793701171875,
      "logps/rejected": -352.86492919921875,
      "loss": 0.0124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8785334825515747,
      "rewards/margins": 9.093118667602539,
      "rewards/rejected": -9.971652030944824,
      "step": 2090
    },
    {
      "epoch": 2.17,
      "learning_rate": 1.5424311926605506e-07,
      "logits/chosen": -2.6098523139953613,
      "logits/rejected": -2.3775291442871094,
      "logps/chosen": -197.1171112060547,
      "logps/rejected": -261.989990234375,
      "loss": 0.0134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.098048686981201,
      "rewards/margins": 7.947665214538574,
      "rewards/rejected": -10.045713424682617,
      "step": 2100
    },
    {
      "epoch": 2.17,
      "eval_logits/chosen": -2.8318982124328613,
      "eval_logits/rejected": -2.7316362857818604,
      "eval_logps/chosen": -283.78375244140625,
      "eval_logps/rejected": -260.8829345703125,
      "eval_loss": 0.624667227268219,
      "eval_rewards/accuracies": 0.796875,
      "eval_rewards/chosen": -2.9002106189727783,
      "eval_rewards/margins": 3.5274696350097656,
      "eval_rewards/rejected": -6.427680015563965,
      "eval_runtime": 43.0871,
      "eval_samples_per_second": 46.418,
      "eval_steps_per_second": 0.371,
      "step": 2100
    },
    {
      "epoch": 2.18,
      "learning_rate": 1.5233180428134556e-07,
      "logits/chosen": -2.7438690662384033,
      "logits/rejected": -2.7524821758270264,
      "logps/chosen": -290.9019470214844,
      "logps/rejected": -347.4677734375,
      "loss": 0.0134,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1246836185455322,
      "rewards/margins": 7.351356506347656,
      "rewards/rejected": -8.47603988647461,
      "step": 2110
    },
    {
      "epoch": 2.19,
      "learning_rate": 1.504204892966361e-07,
      "logits/chosen": -2.690046548843384,
      "logits/rejected": -2.62469482421875,
      "logps/chosen": -311.0474548339844,
      "logps/rejected": -298.30712890625,
      "loss": 0.0154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0858474969863892,
      "rewards/margins": 8.117149353027344,
      "rewards/rejected": -9.202996253967285,
      "step": 2120
    },
    {
      "epoch": 2.2,
      "learning_rate": 1.485091743119266e-07,
      "logits/chosen": -2.8562185764312744,
      "logits/rejected": -2.830827236175537,
      "logps/chosen": -255.25137329101562,
      "logps/rejected": -300.23699951171875,
      "loss": 0.012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9563596844673157,
      "rewards/margins": 7.325954437255859,
      "rewards/rejected": -8.282313346862793,
      "step": 2130
    },
    {
      "epoch": 2.21,
      "learning_rate": 1.465978593272171e-07,
      "logits/chosen": -2.6940760612487793,
      "logits/rejected": -2.521862506866455,
      "logps/chosen": -249.3166961669922,
      "logps/rejected": -279.20245361328125,
      "loss": 0.0122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.594799518585205,
      "rewards/margins": 8.052389144897461,
      "rewards/rejected": -9.64719009399414,
      "step": 2140
    },
    {
      "epoch": 2.22,
      "learning_rate": 1.4468654434250764e-07,
      "logits/chosen": -2.561811923980713,
      "logits/rejected": -2.610355854034424,
      "logps/chosen": -373.43359375,
      "logps/rejected": -411.88275146484375,
      "loss": 0.0154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.011890411376953125,
      "rewards/margins": 9.87723159790039,
      "rewards/rejected": -9.88912296295166,
      "step": 2150
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.4277522935779814e-07,
      "logits/chosen": -2.9392905235290527,
      "logits/rejected": -2.6764588356018066,
      "logps/chosen": -287.12225341796875,
      "logps/rejected": -307.7725524902344,
      "loss": 0.0138,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.639965832233429,
      "rewards/margins": 8.369826316833496,
      "rewards/rejected": -9.009793281555176,
      "step": 2160
    },
    {
      "epoch": 2.24,
      "learning_rate": 1.4086391437308867e-07,
      "logits/chosen": -2.7557687759399414,
      "logits/rejected": -2.8420677185058594,
      "logps/chosen": -300.71356201171875,
      "logps/rejected": -359.48260498046875,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5477391481399536,
      "rewards/margins": 8.222277641296387,
      "rewards/rejected": -9.770015716552734,
      "step": 2170
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.389525993883792e-07,
      "logits/chosen": -2.740018844604492,
      "logits/rejected": -2.632035970687866,
      "logps/chosen": -366.1664733886719,
      "logps/rejected": -380.9911193847656,
      "loss": 0.0142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1040074825286865,
      "rewards/margins": 9.593006134033203,
      "rewards/rejected": -10.697013854980469,
      "step": 2180
    },
    {
      "epoch": 2.26,
      "learning_rate": 1.370412844036697e-07,
      "logits/chosen": -2.78129243850708,
      "logits/rejected": -2.6934008598327637,
      "logps/chosen": -292.3312072753906,
      "logps/rejected": -350.51715087890625,
      "loss": 0.0124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5786378383636475,
      "rewards/margins": 9.707735061645508,
      "rewards/rejected": -10.286372184753418,
      "step": 2190
    },
    {
      "epoch": 2.27,
      "learning_rate": 1.3512996941896024e-07,
      "logits/chosen": -2.6506826877593994,
      "logits/rejected": -2.67533016204834,
      "logps/chosen": -273.0309143066406,
      "logps/rejected": -347.6629333496094,
      "loss": 0.0148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5998280048370361,
      "rewards/margins": 7.78562068939209,
      "rewards/rejected": -9.385449409484863,
      "step": 2200
    },
    {
      "epoch": 2.27,
      "eval_logits/chosen": -2.8063864707946777,
      "eval_logits/rejected": -2.6991143226623535,
      "eval_logps/chosen": -287.3020324707031,
      "eval_logps/rejected": -267.23297119140625,
      "eval_loss": 0.6402204632759094,
      "eval_rewards/accuracies": 0.78125,
      "eval_rewards/chosen": -3.2520391941070557,
      "eval_rewards/margins": 3.8106439113616943,
      "eval_rewards/rejected": -7.062682628631592,
      "eval_runtime": 42.9356,
      "eval_samples_per_second": 46.581,
      "eval_steps_per_second": 0.373,
      "step": 2200
    },
    {
      "epoch": 2.28,
      "learning_rate": 1.3321865443425075e-07,
      "logits/chosen": -2.5518293380737305,
      "logits/rejected": -2.3668487071990967,
      "logps/chosen": -313.5392761230469,
      "logps/rejected": -260.40521240234375,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6749378442764282,
      "rewards/margins": 8.3352632522583,
      "rewards/rejected": -9.010199546813965,
      "step": 2210
    },
    {
      "epoch": 2.29,
      "learning_rate": 1.3130733944954128e-07,
      "logits/chosen": -2.58048677444458,
      "logits/rejected": -2.2754364013671875,
      "logps/chosen": -338.11065673828125,
      "logps/rejected": -319.00518798828125,
      "loss": 0.0113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6327890157699585,
      "rewards/margins": 8.703729629516602,
      "rewards/rejected": -9.336518287658691,
      "step": 2220
    },
    {
      "epoch": 2.3,
      "learning_rate": 1.293960244648318e-07,
      "logits/chosen": -2.8132143020629883,
      "logits/rejected": -2.7200472354888916,
      "logps/chosen": -389.08538818359375,
      "logps/rejected": -325.16363525390625,
      "loss": 0.0125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.311343789100647,
      "rewards/margins": 8.575726509094238,
      "rewards/rejected": -9.887070655822754,
      "step": 2230
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.2748470948012232e-07,
      "logits/chosen": -2.630483865737915,
      "logits/rejected": -2.5980231761932373,
      "logps/chosen": -340.43365478515625,
      "logps/rejected": -373.0090637207031,
      "loss": 0.0068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6955748796463013,
      "rewards/margins": 8.990997314453125,
      "rewards/rejected": -9.686572074890137,
      "step": 2240
    },
    {
      "epoch": 2.32,
      "learning_rate": 1.2557339449541285e-07,
      "logits/chosen": -2.6748175621032715,
      "logits/rejected": -2.5872254371643066,
      "logps/chosen": -335.07373046875,
      "logps/rejected": -352.5212097167969,
      "loss": 0.0202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3692783117294312,
      "rewards/margins": 9.16440200805664,
      "rewards/rejected": -10.53368091583252,
      "step": 2250
    },
    {
      "epoch": 2.33,
      "learning_rate": 1.2366207951070336e-07,
      "logits/chosen": -2.8568575382232666,
      "logits/rejected": -2.7505245208740234,
      "logps/chosen": -275.38433837890625,
      "logps/rejected": -354.7310485839844,
      "loss": 0.0111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.296015739440918,
      "rewards/margins": 11.606992721557617,
      "rewards/rejected": -12.903007507324219,
      "step": 2260
    },
    {
      "epoch": 2.34,
      "learning_rate": 1.217507645259939e-07,
      "logits/chosen": -2.8438689708709717,
      "logits/rejected": -2.772092342376709,
      "logps/chosen": -342.33770751953125,
      "logps/rejected": -421.205078125,
      "loss": 0.0052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9183975458145142,
      "rewards/margins": 9.3962984085083,
      "rewards/rejected": -10.314695358276367,
      "step": 2270
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.198394495412844e-07,
      "logits/chosen": -2.9177348613739014,
      "logits/rejected": -2.8442885875701904,
      "logps/chosen": -354.0315856933594,
      "logps/rejected": -350.19561767578125,
      "loss": 0.0155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20095424354076385,
      "rewards/margins": 8.697885513305664,
      "rewards/rejected": -8.898839950561523,
      "step": 2280
    },
    {
      "epoch": 2.36,
      "learning_rate": 1.1792813455657493e-07,
      "logits/chosen": -2.6511244773864746,
      "logits/rejected": -2.679464340209961,
      "logps/chosen": -277.9000244140625,
      "logps/rejected": -424.24658203125,
      "loss": 0.0193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.517256736755371,
      "rewards/margins": 10.369354248046875,
      "rewards/rejected": -11.886610984802246,
      "step": 2290
    },
    {
      "epoch": 2.37,
      "learning_rate": 1.1601681957186543e-07,
      "logits/chosen": -2.781261920928955,
      "logits/rejected": -2.7080893516540527,
      "logps/chosen": -325.17877197265625,
      "logps/rejected": -291.9747619628906,
      "loss": 0.0142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1653848886489868,
      "rewards/margins": 8.179758071899414,
      "rewards/rejected": -9.34514331817627,
      "step": 2300
    },
    {
      "epoch": 2.37,
      "eval_logits/chosen": -2.799234628677368,
      "eval_logits/rejected": -2.6871373653411865,
      "eval_logps/chosen": -287.49615478515625,
      "eval_logps/rejected": -267.9088134765625,
      "eval_loss": 0.6562942862510681,
      "eval_rewards/accuracies": 0.828125,
      "eval_rewards/chosen": -3.2714524269104004,
      "eval_rewards/margins": 3.8588109016418457,
      "eval_rewards/rejected": -7.130263328552246,
      "eval_runtime": 42.8613,
      "eval_samples_per_second": 46.662,
      "eval_steps_per_second": 0.373,
      "step": 2300
    },
    {
      "epoch": 2.38,
      "learning_rate": 1.1410550458715595e-07,
      "logits/chosen": -2.5371177196502686,
      "logits/rejected": -2.641981601715088,
      "logps/chosen": -329.0419006347656,
      "logps/rejected": -356.4971618652344,
      "loss": 0.0163,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5353990793228149,
      "rewards/margins": 8.938331604003906,
      "rewards/rejected": -9.473730087280273,
      "step": 2310
    },
    {
      "epoch": 2.39,
      "learning_rate": 1.1219418960244648e-07,
      "logits/chosen": -2.671187400817871,
      "logits/rejected": -2.650254249572754,
      "logps/chosen": -236.1153564453125,
      "logps/rejected": -307.6558532714844,
      "loss": 0.0112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7011358737945557,
      "rewards/margins": 8.071800231933594,
      "rewards/rejected": -9.77293586730957,
      "step": 2320
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.10282874617737e-07,
      "logits/chosen": -2.7763831615448,
      "logits/rejected": -2.5476737022399902,
      "logps/chosen": -263.58221435546875,
      "logps/rejected": -328.7640075683594,
      "loss": 0.0139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8821464776992798,
      "rewards/margins": 7.959317207336426,
      "rewards/rejected": -9.841463088989258,
      "step": 2330
    },
    {
      "epoch": 2.41,
      "learning_rate": 1.0837155963302752e-07,
      "logits/chosen": -2.524266242980957,
      "logits/rejected": -2.485341787338257,
      "logps/chosen": -288.97955322265625,
      "logps/rejected": -307.2179870605469,
      "loss": 0.0143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7493270635604858,
      "rewards/margins": 7.521073818206787,
      "rewards/rejected": -9.270401954650879,
      "step": 2340
    },
    {
      "epoch": 2.43,
      "learning_rate": 1.0646024464831804e-07,
      "logits/chosen": -2.592519998550415,
      "logits/rejected": -2.5024571418762207,
      "logps/chosen": -289.25762939453125,
      "logps/rejected": -368.61883544921875,
      "loss": 0.0148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6334786415100098,
      "rewards/margins": 9.766633033752441,
      "rewards/rejected": -12.400110244750977,
      "step": 2350
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.0454892966360856e-07,
      "logits/chosen": -2.598473072052002,
      "logits/rejected": -2.5985465049743652,
      "logps/chosen": -300.30975341796875,
      "logps/rejected": -309.1223449707031,
      "loss": 0.01,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0343241691589355,
      "rewards/margins": 9.686413764953613,
      "rewards/rejected": -11.720738410949707,
      "step": 2360
    },
    {
      "epoch": 2.45,
      "learning_rate": 1.0263761467889908e-07,
      "logits/chosen": -2.6408867835998535,
      "logits/rejected": -2.503786087036133,
      "logps/chosen": -291.9281005859375,
      "logps/rejected": -322.25714111328125,
      "loss": 0.0123,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5202815532684326,
      "rewards/margins": 9.04098129272461,
      "rewards/rejected": -9.561263084411621,
      "step": 2370
    },
    {
      "epoch": 2.46,
      "learning_rate": 1.007262996941896e-07,
      "logits/chosen": -2.3971962928771973,
      "logits/rejected": -2.137141466140747,
      "logps/chosen": -314.1961364746094,
      "logps/rejected": -306.4624328613281,
      "loss": 0.0144,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.843491792678833,
      "rewards/margins": 8.445917129516602,
      "rewards/rejected": -10.289408683776855,
      "step": 2380
    },
    {
      "epoch": 2.47,
      "learning_rate": 9.881498470948011e-08,
      "logits/chosen": -2.681727409362793,
      "logits/rejected": -2.449688673019409,
      "logps/chosen": -270.8292541503906,
      "logps/rejected": -305.0626525878906,
      "loss": 0.0124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.713470220565796,
      "rewards/margins": 7.436466217041016,
      "rewards/rejected": -10.14993667602539,
      "step": 2390
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.690366972477065e-08,
      "logits/chosen": -2.7360880374908447,
      "logits/rejected": -2.639930248260498,
      "logps/chosen": -212.0613250732422,
      "logps/rejected": -236.4873504638672,
      "loss": 0.011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5846588611602783,
      "rewards/margins": 8.755882263183594,
      "rewards/rejected": -10.34054183959961,
      "step": 2400
    },
    {
      "epoch": 2.48,
      "eval_logits/chosen": -2.7717225551605225,
      "eval_logits/rejected": -2.655519723892212,
      "eval_logps/chosen": -287.77764892578125,
      "eval_logps/rejected": -268.86431884765625,
      "eval_loss": 0.6604524254798889,
      "eval_rewards/accuracies": 0.796875,
      "eval_rewards/chosen": -3.2996015548706055,
      "eval_rewards/margins": 3.926215648651123,
      "eval_rewards/rejected": -7.22581672668457,
      "eval_runtime": 42.9016,
      "eval_samples_per_second": 46.618,
      "eval_steps_per_second": 0.373,
      "step": 2400
    },
    {
      "epoch": 2.49,
      "learning_rate": 9.499235474006116e-08,
      "logits/chosen": -2.685112237930298,
      "logits/rejected": -2.5567049980163574,
      "logps/chosen": -453.57830810546875,
      "logps/rejected": -347.819091796875,
      "loss": 0.018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1265848875045776,
      "rewards/margins": 10.750268936157227,
      "rewards/rejected": -11.876853942871094,
      "step": 2410
    },
    {
      "epoch": 2.5,
      "learning_rate": 9.308103975535168e-08,
      "logits/chosen": -2.835233211517334,
      "logits/rejected": -2.7287702560424805,
      "logps/chosen": -341.2040710449219,
      "logps/rejected": -335.99957275390625,
      "loss": 0.0146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6093537211418152,
      "rewards/margins": 8.289168357849121,
      "rewards/rejected": -8.898521423339844,
      "step": 2420
    },
    {
      "epoch": 2.51,
      "learning_rate": 9.116972477064219e-08,
      "logits/chosen": -2.782958507537842,
      "logits/rejected": -2.7175862789154053,
      "logps/chosen": -376.49114990234375,
      "logps/rejected": -467.7449645996094,
      "loss": 0.01,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5592962503433228,
      "rewards/margins": 8.31694221496582,
      "rewards/rejected": -9.876237869262695,
      "step": 2430
    },
    {
      "epoch": 2.52,
      "learning_rate": 8.925840978593272e-08,
      "logits/chosen": -2.642646312713623,
      "logits/rejected": -2.5226454734802246,
      "logps/chosen": -199.7229461669922,
      "logps/rejected": -280.9407043457031,
      "loss": 0.0167,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.0735208988189697,
      "rewards/margins": 8.644987106323242,
      "rewards/rejected": -9.718507766723633,
      "step": 2440
    },
    {
      "epoch": 2.53,
      "learning_rate": 8.734709480122324e-08,
      "logits/chosen": -2.500033378601074,
      "logits/rejected": -2.669753074645996,
      "logps/chosen": -184.5748748779297,
      "logps/rejected": -312.9557189941406,
      "loss": 0.0128,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.3214828968048096,
      "rewards/margins": 7.528273105621338,
      "rewards/rejected": -9.849756240844727,
      "step": 2450
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.543577981651376e-08,
      "logits/chosen": -2.832641124725342,
      "logits/rejected": -2.600156545639038,
      "logps/chosen": -452.07177734375,
      "logps/rejected": -371.8671569824219,
      "loss": 0.0126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0375783443450928,
      "rewards/margins": 9.188015937805176,
      "rewards/rejected": -10.225595474243164,
      "step": 2460
    },
    {
      "epoch": 2.55,
      "learning_rate": 8.352446483180428e-08,
      "logits/chosen": -2.801546096801758,
      "logits/rejected": -2.646188259124756,
      "logps/chosen": -281.76971435546875,
      "logps/rejected": -274.42327880859375,
      "loss": 0.0227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7577645182609558,
      "rewards/margins": 8.514206886291504,
      "rewards/rejected": -9.271970748901367,
      "step": 2470
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.161314984709481e-08,
      "logits/chosen": -2.8526995182037354,
      "logits/rejected": -2.7566418647766113,
      "logps/chosen": -354.4234924316406,
      "logps/rejected": -326.2621765136719,
      "loss": 0.0047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5101653933525085,
      "rewards/margins": 9.834528923034668,
      "rewards/rejected": -10.344694137573242,
      "step": 2480
    },
    {
      "epoch": 2.57,
      "learning_rate": 7.970183486238531e-08,
      "logits/chosen": -2.714329957962036,
      "logits/rejected": -2.7505898475646973,
      "logps/chosen": -221.5650177001953,
      "logps/rejected": -294.24505615234375,
      "loss": 0.012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7582021951675415,
      "rewards/margins": 8.098190307617188,
      "rewards/rejected": -9.856393814086914,
      "step": 2490
    },
    {
      "epoch": 2.58,
      "learning_rate": 7.779051987767583e-08,
      "logits/chosen": -2.8804407119750977,
      "logits/rejected": -2.6511168479919434,
      "logps/chosen": -427.35406494140625,
      "logps/rejected": -327.26007080078125,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4930397272109985,
      "rewards/margins": 9.664579391479492,
      "rewards/rejected": -11.15761947631836,
      "step": 2500
    },
    {
      "epoch": 2.58,
      "eval_logits/chosen": -2.7902204990386963,
      "eval_logits/rejected": -2.6779799461364746,
      "eval_logps/chosen": -291.1807861328125,
      "eval_logps/rejected": -276.8377380371094,
      "eval_loss": 0.6934967041015625,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -3.639915704727173,
      "eval_rewards/margins": 4.383244514465332,
      "eval_rewards/rejected": -8.023159980773926,
      "eval_runtime": 43.4056,
      "eval_samples_per_second": 46.077,
      "eval_steps_per_second": 0.369,
      "step": 2500
    },
    {
      "epoch": 2.59,
      "learning_rate": 7.587920489296635e-08,
      "logits/chosen": -2.53355073928833,
      "logits/rejected": -2.469438076019287,
      "logps/chosen": -219.6329803466797,
      "logps/rejected": -303.95587158203125,
      "loss": 0.0112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3920328617095947,
      "rewards/margins": 9.254063606262207,
      "rewards/rejected": -10.646096229553223,
      "step": 2510
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.396788990825688e-08,
      "logits/chosen": -2.6555113792419434,
      "logits/rejected": -2.589629888534546,
      "logps/chosen": -236.41653442382812,
      "logps/rejected": -295.16082763671875,
      "loss": 0.015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.670567035675049,
      "rewards/margins": 7.430187225341797,
      "rewards/rejected": -10.100754737854004,
      "step": 2520
    },
    {
      "epoch": 2.61,
      "learning_rate": 7.20565749235474e-08,
      "logits/chosen": -2.6323156356811523,
      "logits/rejected": -2.664762258529663,
      "logps/chosen": -193.96356201171875,
      "logps/rejected": -328.59417724609375,
      "loss": 0.0171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1080195903778076,
      "rewards/margins": 8.095452308654785,
      "rewards/rejected": -9.203471183776855,
      "step": 2530
    },
    {
      "epoch": 2.62,
      "learning_rate": 7.014525993883792e-08,
      "logits/chosen": -2.6163525581359863,
      "logits/rejected": -2.47815203666687,
      "logps/chosen": -309.06378173828125,
      "logps/rejected": -324.7481689453125,
      "loss": 0.0136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1383777856826782,
      "rewards/margins": 8.445047378540039,
      "rewards/rejected": -9.583425521850586,
      "step": 2540
    },
    {
      "epoch": 2.63,
      "learning_rate": 6.823394495412843e-08,
      "logits/chosen": -2.5601565837860107,
      "logits/rejected": -2.7058122158050537,
      "logps/chosen": -250.9035186767578,
      "logps/rejected": -373.5662536621094,
      "loss": 0.013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8016271591186523,
      "rewards/margins": 12.705131530761719,
      "rewards/rejected": -13.506759643554688,
      "step": 2550
    },
    {
      "epoch": 2.64,
      "learning_rate": 6.632262996941895e-08,
      "logits/chosen": -2.721140146255493,
      "logits/rejected": -2.4993348121643066,
      "logps/chosen": -321.29547119140625,
      "logps/rejected": -343.4897155761719,
      "loss": 0.0097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9347482919692993,
      "rewards/margins": 9.293264389038086,
      "rewards/rejected": -11.228011131286621,
      "step": 2560
    },
    {
      "epoch": 2.65,
      "learning_rate": 6.441131498470948e-08,
      "logits/chosen": -2.821812391281128,
      "logits/rejected": -2.5831847190856934,
      "logps/chosen": -294.45684814453125,
      "logps/rejected": -276.64080810546875,
      "loss": 0.0204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6778271198272705,
      "rewards/margins": 7.781607151031494,
      "rewards/rejected": -9.459434509277344,
      "step": 2570
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.25e-08,
      "logits/chosen": -2.762652635574341,
      "logits/rejected": -2.630335569381714,
      "logps/chosen": -346.9800720214844,
      "logps/rejected": -324.47760009765625,
      "loss": 0.0121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6068371534347534,
      "rewards/margins": 8.518617630004883,
      "rewards/rejected": -9.125454902648926,
      "step": 2580
    },
    {
      "epoch": 2.67,
      "learning_rate": 6.058868501529052e-08,
      "logits/chosen": -2.796215534210205,
      "logits/rejected": -2.7548739910125732,
      "logps/chosen": -265.49566650390625,
      "logps/rejected": -304.27313232421875,
      "loss": 0.0163,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0402780771255493,
      "rewards/margins": 9.19719123840332,
      "rewards/rejected": -10.237469673156738,
      "step": 2590
    },
    {
      "epoch": 2.68,
      "learning_rate": 5.8677370030581035e-08,
      "logits/chosen": -2.604027271270752,
      "logits/rejected": -2.4126858711242676,
      "logps/chosen": -307.8719787597656,
      "logps/rejected": -329.3260803222656,
      "loss": 0.0089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9125175476074219,
      "rewards/margins": 9.744293212890625,
      "rewards/rejected": -10.65680980682373,
      "step": 2600
    },
    {
      "epoch": 2.68,
      "eval_logits/chosen": -2.7994275093078613,
      "eval_logits/rejected": -2.6884756088256836,
      "eval_logps/chosen": -289.603271484375,
      "eval_logps/rejected": -274.7880554199219,
      "eval_loss": 0.6772685647010803,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -3.4821622371673584,
      "eval_rewards/margins": 4.336028099060059,
      "eval_rewards/rejected": -7.818190574645996,
      "eval_runtime": 42.7413,
      "eval_samples_per_second": 46.793,
      "eval_steps_per_second": 0.374,
      "step": 2600
    },
    {
      "epoch": 2.69,
      "learning_rate": 5.6766055045871554e-08,
      "logits/chosen": -2.764072895050049,
      "logits/rejected": -2.712106227874756,
      "logps/chosen": -356.59771728515625,
      "logps/rejected": -422.31121826171875,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.934819221496582,
      "rewards/margins": 10.432962417602539,
      "rewards/rejected": -11.367780685424805,
      "step": 2610
    },
    {
      "epoch": 2.7,
      "learning_rate": 5.485474006116208e-08,
      "logits/chosen": -2.730092763900757,
      "logits/rejected": -2.6763250827789307,
      "logps/chosen": -278.66619873046875,
      "logps/rejected": -317.0282287597656,
      "loss": 0.0105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1639602184295654,
      "rewards/margins": 8.831220626831055,
      "rewards/rejected": -9.9951810836792,
      "step": 2620
    },
    {
      "epoch": 2.71,
      "learning_rate": 5.294342507645259e-08,
      "logits/chosen": -2.771960496902466,
      "logits/rejected": -2.5834813117980957,
      "logps/chosen": -342.79241943359375,
      "logps/rejected": -364.6011657714844,
      "loss": 0.0067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05748433992266655,
      "rewards/margins": 10.371294975280762,
      "rewards/rejected": -10.313809394836426,
      "step": 2630
    },
    {
      "epoch": 2.72,
      "learning_rate": 5.1032110091743117e-08,
      "logits/chosen": -2.6307904720306396,
      "logits/rejected": -2.3652987480163574,
      "logps/chosen": -281.32806396484375,
      "logps/rejected": -276.0794677734375,
      "loss": 0.0079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8738715648651123,
      "rewards/margins": 7.841960906982422,
      "rewards/rejected": -9.715832710266113,
      "step": 2640
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.9120795107033635e-08,
      "logits/chosen": -2.741020441055298,
      "logits/rejected": -2.4758517742156982,
      "logps/chosen": -283.36956787109375,
      "logps/rejected": -374.82598876953125,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3181073665618896,
      "rewards/margins": 10.364333152770996,
      "rewards/rejected": -11.682439804077148,
      "step": 2650
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.7209480122324154e-08,
      "logits/chosen": -2.8300743103027344,
      "logits/rejected": -2.601854085922241,
      "logps/chosen": -338.0480041503906,
      "logps/rejected": -443.51226806640625,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.905073881149292,
      "rewards/margins": 9.982545852661133,
      "rewards/rejected": -12.88762092590332,
      "step": 2660
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.529816513761467e-08,
      "logits/chosen": -2.666491985321045,
      "logits/rejected": -2.4424288272857666,
      "logps/chosen": -267.314208984375,
      "logps/rejected": -384.9258117675781,
      "loss": 0.0121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7637933492660522,
      "rewards/margins": 8.94968318939209,
      "rewards/rejected": -10.713478088378906,
      "step": 2670
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.33868501529052e-08,
      "logits/chosen": -2.690896511077881,
      "logits/rejected": -2.5063717365264893,
      "logps/chosen": -264.0552978515625,
      "logps/rejected": -322.1650085449219,
      "loss": 0.0175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4952847957611084,
      "rewards/margins": 10.171586990356445,
      "rewards/rejected": -11.666872024536133,
      "step": 2680
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.147553516819572e-08,
      "logits/chosen": -2.720097780227661,
      "logits/rejected": -2.623161792755127,
      "logps/chosen": -245.23593139648438,
      "logps/rejected": -336.7532653808594,
      "loss": 0.0093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.500422477722168,
      "rewards/margins": 8.012624740600586,
      "rewards/rejected": -10.513047218322754,
      "step": 2690
    },
    {
      "epoch": 2.79,
      "learning_rate": 3.9564220183486236e-08,
      "logits/chosen": -2.734628438949585,
      "logits/rejected": -2.626295804977417,
      "logps/chosen": -258.6006164550781,
      "logps/rejected": -283.94329833984375,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5415168404579163,
      "rewards/margins": 9.378320693969727,
      "rewards/rejected": -9.919837951660156,
      "step": 2700
    },
    {
      "epoch": 2.79,
      "eval_logits/chosen": -2.797022819519043,
      "eval_logits/rejected": -2.687746286392212,
      "eval_logps/chosen": -290.6907958984375,
      "eval_logps/rejected": -274.7027587890625,
      "eval_loss": 0.6812653541564941,
      "eval_rewards/accuracies": 0.828125,
      "eval_rewards/chosen": -3.590911865234375,
      "eval_rewards/margins": 4.218749523162842,
      "eval_rewards/rejected": -7.809660911560059,
      "eval_runtime": 43.3751,
      "eval_samples_per_second": 46.109,
      "eval_steps_per_second": 0.369,
      "step": 2700
    },
    {
      "epoch": 2.8,
      "learning_rate": 3.7652905198776755e-08,
      "logits/chosen": -2.766024351119995,
      "logits/rejected": -2.5780766010284424,
      "logps/chosen": -317.5596618652344,
      "logps/rejected": -323.1799011230469,
      "loss": 0.0095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4215131998062134,
      "rewards/margins": 7.726994514465332,
      "rewards/rejected": -9.148508071899414,
      "step": 2710
    },
    {
      "epoch": 2.81,
      "learning_rate": 3.574159021406728e-08,
      "logits/chosen": -2.7624149322509766,
      "logits/rejected": -2.6529033184051514,
      "logps/chosen": -281.3433837890625,
      "logps/rejected": -344.85369873046875,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8688713312149048,
      "rewards/margins": 9.345159530639648,
      "rewards/rejected": -11.214031219482422,
      "step": 2720
    },
    {
      "epoch": 2.82,
      "learning_rate": 3.383027522935779e-08,
      "logits/chosen": -2.845158338546753,
      "logits/rejected": -2.767922878265381,
      "logps/chosen": -278.7703857421875,
      "logps/rejected": -385.30291748046875,
      "loss": 0.0107,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.0106799602508545,
      "rewards/margins": 10.258461952209473,
      "rewards/rejected": -11.26914119720459,
      "step": 2730
    },
    {
      "epoch": 2.83,
      "learning_rate": 3.191896024464832e-08,
      "logits/chosen": -2.7531659603118896,
      "logits/rejected": -2.599841594696045,
      "logps/chosen": -311.2244873046875,
      "logps/rejected": -315.21246337890625,
      "loss": 0.008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4379760026931763,
      "rewards/margins": 10.220308303833008,
      "rewards/rejected": -11.658284187316895,
      "step": 2740
    },
    {
      "epoch": 2.84,
      "learning_rate": 3.0007645259938836e-08,
      "logits/chosen": -2.6417484283447266,
      "logits/rejected": -2.6449437141418457,
      "logps/chosen": -268.1544189453125,
      "logps/rejected": -399.4880676269531,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2721059322357178,
      "rewards/margins": 14.419519424438477,
      "rewards/rejected": -12.147417068481445,
      "step": 2750
    },
    {
      "epoch": 2.85,
      "learning_rate": 2.809633027522936e-08,
      "logits/chosen": -2.5494630336761475,
      "logits/rejected": -2.4989705085754395,
      "logps/chosen": -230.28109741210938,
      "logps/rejected": -325.3707580566406,
      "loss": 0.015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.975464344024658,
      "rewards/margins": 7.614091396331787,
      "rewards/rejected": -10.589555740356445,
      "step": 2760
    },
    {
      "epoch": 2.86,
      "learning_rate": 2.6185015290519877e-08,
      "logits/chosen": -2.411627769470215,
      "logits/rejected": -2.542217493057251,
      "logps/chosen": -201.55873107910156,
      "logps/rejected": -297.8031005859375,
      "loss": 0.01,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.786314606666565,
      "rewards/margins": 9.065244674682617,
      "rewards/rejected": -10.85155963897705,
      "step": 2770
    },
    {
      "epoch": 2.87,
      "learning_rate": 2.4273700305810396e-08,
      "logits/chosen": -2.826395034790039,
      "logits/rejected": -2.7339437007904053,
      "logps/chosen": -320.2567443847656,
      "logps/rejected": -365.04473876953125,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3468475341796875,
      "rewards/margins": 10.051078796386719,
      "rewards/rejected": -12.39792537689209,
      "step": 2780
    },
    {
      "epoch": 2.88,
      "learning_rate": 2.2362385321100918e-08,
      "logits/chosen": -2.5493578910827637,
      "logits/rejected": -2.556107997894287,
      "logps/chosen": -293.81561279296875,
      "logps/rejected": -364.3607482910156,
      "loss": 0.0134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6632717251777649,
      "rewards/margins": 10.876815795898438,
      "rewards/rejected": -11.540087699890137,
      "step": 2790
    },
    {
      "epoch": 2.89,
      "learning_rate": 2.0451070336391437e-08,
      "logits/chosen": -2.2315673828125,
      "logits/rejected": -2.153177499771118,
      "logps/chosen": -197.33572387695312,
      "logps/rejected": -307.0080261230469,
      "loss": 0.0136,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.3446338474750519,
      "rewards/margins": 11.82870864868164,
      "rewards/rejected": -11.484074592590332,
      "step": 2800
    },
    {
      "epoch": 2.89,
      "eval_logits/chosen": -2.786235809326172,
      "eval_logits/rejected": -2.6765427589416504,
      "eval_logps/chosen": -293.01751708984375,
      "eval_logps/rejected": -278.0957336425781,
      "eval_loss": 0.6891822218894958,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -3.8235886096954346,
      "eval_rewards/margins": 4.3253679275512695,
      "eval_rewards/rejected": -8.148956298828125,
      "eval_runtime": 43.4023,
      "eval_samples_per_second": 46.081,
      "eval_steps_per_second": 0.369,
      "step": 2800
    },
    {
      "epoch": 2.9,
      "learning_rate": 1.8539755351681956e-08,
      "logits/chosen": -2.6543760299682617,
      "logits/rejected": -2.726107358932495,
      "logps/chosen": -423.95465087890625,
      "logps/rejected": -416.55419921875,
      "loss": 0.0164,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1608713865280151,
      "rewards/margins": 9.049515724182129,
      "rewards/rejected": -10.210386276245117,
      "step": 2810
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.6628440366972478e-08,
      "logits/chosen": -2.6333303451538086,
      "logits/rejected": -2.356459856033325,
      "logps/chosen": -344.8193054199219,
      "logps/rejected": -257.2393493652344,
      "loss": 0.0119,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.881445288658142,
      "rewards/margins": 8.154787063598633,
      "rewards/rejected": -10.036233901977539,
      "step": 2820
    },
    {
      "epoch": 2.92,
      "learning_rate": 1.4717125382262997e-08,
      "logits/chosen": -2.818207025527954,
      "logits/rejected": -2.738051176071167,
      "logps/chosen": -335.53314208984375,
      "logps/rejected": -399.3725891113281,
      "loss": 0.011,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7855017185211182,
      "rewards/margins": 8.13086223602295,
      "rewards/rejected": -9.916363716125488,
      "step": 2830
    },
    {
      "epoch": 2.93,
      "learning_rate": 1.2805810397553517e-08,
      "logits/chosen": -2.5987372398376465,
      "logits/rejected": -2.5041844844818115,
      "logps/chosen": -395.9434509277344,
      "logps/rejected": -483.74322509765625,
      "loss": 0.0152,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.267432689666748,
      "rewards/margins": 10.311139106750488,
      "rewards/rejected": -12.578573226928711,
      "step": 2840
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.0894495412844038e-08,
      "logits/chosen": -2.592224597930908,
      "logits/rejected": -2.6307315826416016,
      "logps/chosen": -259.36126708984375,
      "logps/rejected": -329.85626220703125,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8243284225463867,
      "rewards/margins": 8.955255508422852,
      "rewards/rejected": -9.779583930969238,
      "step": 2850
    },
    {
      "epoch": 2.95,
      "learning_rate": 8.983180428134555e-09,
      "logits/chosen": -2.685051441192627,
      "logits/rejected": -2.527430295944214,
      "logps/chosen": -244.1125946044922,
      "logps/rejected": -238.81472778320312,
      "loss": 0.0119,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.916625738143921,
      "rewards/margins": 6.8498854637146,
      "rewards/rejected": -9.766510963439941,
      "step": 2860
    },
    {
      "epoch": 2.96,
      "learning_rate": 7.071865443425076e-09,
      "logits/chosen": -2.5415682792663574,
      "logits/rejected": -2.7124288082122803,
      "logps/chosen": -298.91815185546875,
      "logps/rejected": -437.7132873535156,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.020925998687744,
      "rewards/margins": 9.953208923339844,
      "rewards/rejected": -11.97413444519043,
      "step": 2870
    },
    {
      "epoch": 2.97,
      "learning_rate": 5.1605504587155965e-09,
      "logits/chosen": -2.8062963485717773,
      "logits/rejected": -2.5917065143585205,
      "logps/chosen": -269.49462890625,
      "logps/rejected": -275.78240966796875,
      "loss": 0.0075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.393111228942871,
      "rewards/margins": 8.628837585449219,
      "rewards/rejected": -10.021946907043457,
      "step": 2880
    },
    {
      "epoch": 2.98,
      "learning_rate": 3.249235474006116e-09,
      "logits/chosen": -2.7617385387420654,
      "logits/rejected": -2.6802616119384766,
      "logps/chosen": -315.0368347167969,
      "logps/rejected": -349.5107116699219,
      "loss": 0.0176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1777875423431396,
      "rewards/margins": 8.661017417907715,
      "rewards/rejected": -11.83880615234375,
      "step": 2890
    },
    {
      "epoch": 2.99,
      "learning_rate": 1.3379204892966359e-09,
      "logits/chosen": -2.701010227203369,
      "logits/rejected": -2.6551761627197266,
      "logps/chosen": -245.7960662841797,
      "logps/rejected": -332.9742736816406,
      "loss": 0.0091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9432682991027832,
      "rewards/margins": 9.252389907836914,
      "rewards/rejected": -10.195657730102539,
      "step": 2900
    },
    {
      "epoch": 2.99,
      "eval_logits/chosen": -2.7830052375793457,
      "eval_logits/rejected": -2.6728432178497314,
      "eval_logps/chosen": -293.980224609375,
      "eval_logps/rejected": -279.6104431152344,
      "eval_loss": 0.6913489699363708,
      "eval_rewards/accuracies": 0.8125,
      "eval_rewards/chosen": -3.9198596477508545,
      "eval_rewards/margins": 4.380568027496338,
      "eval_rewards/rejected": -8.30042839050293,
      "eval_runtime": 43.3515,
      "eval_samples_per_second": 46.134,
      "eval_steps_per_second": 0.369,
      "step": 2900
    },
    {
      "epoch": 3.0,
      "step": 2907,
      "total_flos": 0.0,
      "train_loss": 0.20427082364947516,
      "train_runtime": 9903.6907,
      "train_samples_per_second": 18.771,
      "train_steps_per_second": 0.294
    }
  ],
  "logging_steps": 10,
  "max_steps": 2907,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}