{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994767137624281,
  "eval_steps": 100,
  "global_step": 955,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0010465724751439038,
      "grad_norm": 7.312430627947873,
      "learning_rate": 5.208333333333333e-09,
      "logits/chosen": -2.6544837951660156,
      "logits/rejected": -2.5759358406066895,
      "logps/chosen": -101.20021057128906,
      "logps/rejected": -85.73662567138672,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.010465724751439037,
      "grad_norm": 6.798309843145578,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.5073227882385254,
      "logits/rejected": -2.498267650604248,
      "logps/chosen": -76.74188995361328,
      "logps/rejected": -68.77124786376953,
      "loss": 0.6931,
      "rewards/accuracies": 0.1597222238779068,
      "rewards/chosen": -0.00010638780076988041,
      "rewards/margins": -0.00026712569524534047,
      "rewards/rejected": 0.00016073790902737528,
      "step": 10
    },
    {
      "epoch": 0.020931449502878074,
      "grad_norm": 6.328409074575995,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.5329933166503906,
      "logits/rejected": -2.5121402740478516,
      "logps/chosen": -91.55634307861328,
      "logps/rejected": -97.98811340332031,
      "loss": 0.6931,
      "rewards/accuracies": 0.26875001192092896,
      "rewards/chosen": -8.86881971382536e-05,
      "rewards/margins": 0.0001090427249437198,
      "rewards/rejected": -0.00019773092935793102,
      "step": 20
    },
    {
      "epoch": 0.03139717425431711,
      "grad_norm": 5.987591394147146,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.6352438926696777,
      "logits/rejected": -2.624114513397217,
      "logps/chosen": -74.1455078125,
      "logps/rejected": -74.52520751953125,
      "loss": 0.6929,
      "rewards/accuracies": 0.2562499940395355,
      "rewards/chosen": -0.00024412055790890008,
      "rewards/margins": 0.00047922172234393656,
      "rewards/rejected": -0.0007233422948047519,
      "step": 30
    },
    {
      "epoch": 0.04186289900575615,
      "grad_norm": 6.813717975556312,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.5477070808410645,
      "logits/rejected": -2.495793104171753,
      "logps/chosen": -90.96524810791016,
      "logps/rejected": -89.30764770507812,
      "loss": 0.6925,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -3.622327858465724e-05,
      "rewards/margins": 0.0016909090336412191,
      "rewards/rejected": -0.0017271323595196009,
      "step": 40
    },
    {
      "epoch": 0.052328623757195186,
      "grad_norm": 6.382358089839322,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.5686728954315186,
      "logits/rejected": -2.5408482551574707,
      "logps/chosen": -76.85763549804688,
      "logps/rejected": -78.096923828125,
      "loss": 0.6914,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.002494217362254858,
      "rewards/margins": 0.002928710076957941,
      "rewards/rejected": -0.00043449303484521806,
      "step": 50
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 6.142397539571157,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.520242214202881,
      "logits/rejected": -2.5160062313079834,
      "logps/chosen": -71.42273712158203,
      "logps/rejected": -71.63546752929688,
      "loss": 0.6901,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": 0.011287200264632702,
      "rewards/margins": 0.006170675158500671,
      "rewards/rejected": 0.0051165251061320305,
      "step": 60
    },
    {
      "epoch": 0.07326007326007326,
      "grad_norm": 6.834963526035445,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.462669849395752,
      "logits/rejected": -2.448659658432007,
      "logps/chosen": -72.4560775756836,
      "logps/rejected": -89.7001953125,
      "loss": 0.6863,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.019068170338869095,
      "rewards/margins": 0.014615567401051521,
      "rewards/rejected": 0.0044526029378175735,
      "step": 70
    },
    {
      "epoch": 0.0837257980115123,
      "grad_norm": 7.385617041745493,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.4439797401428223,
      "logits/rejected": -2.4138834476470947,
      "logps/chosen": -81.91383361816406,
      "logps/rejected": -80.78437042236328,
      "loss": 0.6811,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 0.00990500207990408,
      "rewards/margins": 0.032499730587005615,
      "rewards/rejected": -0.02259472757577896,
      "step": 80
    },
    {
      "epoch": 0.09419152276295134,
      "grad_norm": 8.102390740927866,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.3960914611816406,
      "logits/rejected": -2.3794617652893066,
      "logps/chosen": -73.29556274414062,
      "logps/rejected": -76.41487121582031,
      "loss": 0.6784,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.040218498557806015,
      "rewards/margins": 0.016615843400359154,
      "rewards/rejected": -0.056834347546100616,
      "step": 90
    },
    {
      "epoch": 0.10465724751439037,
      "grad_norm": 6.755359569975091,
      "learning_rate": 4.999732492681437e-07,
      "logits/chosen": -2.3556602001190186,
      "logits/rejected": -2.3346455097198486,
      "logps/chosen": -78.03315734863281,
      "logps/rejected": -95.56941223144531,
      "loss": 0.6719,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.005892142653465271,
      "rewards/margins": 0.07231049239635468,
      "rewards/rejected": -0.07820263504981995,
      "step": 100
    },
    {
      "epoch": 0.10465724751439037,
      "eval_logits/chosen": -2.3486523628234863,
      "eval_logits/rejected": -2.3309624195098877,
      "eval_logps/chosen": -73.28648376464844,
      "eval_logps/rejected": -85.4478530883789,
      "eval_loss": 0.6686670184135437,
      "eval_rewards/accuracies": 0.3273809552192688,
      "eval_rewards/chosen": 0.012047496624290943,
      "eval_rewards/margins": 0.05524253472685814,
      "eval_rewards/rejected": -0.04319504275918007,
      "eval_runtime": 113.7223,
      "eval_samples_per_second": 17.587,
      "eval_steps_per_second": 0.554,
      "step": 100
    },
    {
      "epoch": 0.1151229722658294,
      "grad_norm": 13.391566933736375,
      "learning_rate": 4.996723692767926e-07,
      "logits/chosen": -2.406934976577759,
      "logits/rejected": -2.4089908599853516,
      "logps/chosen": -80.33895111083984,
      "logps/rejected": -93.52415466308594,
      "loss": 0.6563,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.034039516001939774,
      "rewards/margins": 0.08607066422700882,
      "rewards/rejected": -0.1201101765036583,
      "step": 110
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 9.646857443256218,
      "learning_rate": 4.990375746213598e-07,
      "logits/chosen": -2.2991995811462402,
      "logits/rejected": -2.265392303466797,
      "logps/chosen": -76.64207458496094,
      "logps/rejected": -94.79474639892578,
      "loss": 0.6601,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.01742006093263626,
      "rewards/margins": 0.11232365667819977,
      "rewards/rejected": -0.12974372506141663,
      "step": 120
    },
    {
      "epoch": 0.1360544217687075,
      "grad_norm": 16.35113879818621,
      "learning_rate": 4.980697142834314e-07,
      "logits/chosen": -2.244642496109009,
      "logits/rejected": -2.2259411811828613,
      "logps/chosen": -65.43486022949219,
      "logps/rejected": -84.06461334228516,
      "loss": 0.66,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": -0.04491991177201271,
      "rewards/margins": 0.0634341612458229,
      "rewards/rejected": -0.10835406929254532,
      "step": 130
    },
    {
      "epoch": 0.14652014652014653,
      "grad_norm": 17.442951288216843,
      "learning_rate": 4.967700826904229e-07,
      "logits/chosen": -2.218097686767578,
      "logits/rejected": -2.2064428329467773,
      "logps/chosen": -109.58231353759766,
      "logps/rejected": -120.42280578613281,
      "loss": 0.6529,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": -0.18489012122154236,
      "rewards/margins": 0.14332745969295502,
      "rewards/rejected": -0.3282175660133362,
      "step": 140
    },
    {
      "epoch": 0.15698587127158556,
      "grad_norm": 21.84882656667421,
      "learning_rate": 4.951404179843962e-07,
      "logits/chosen": -2.3305749893188477,
      "logits/rejected": -2.3431344032287598,
      "logps/chosen": -69.88795471191406,
      "logps/rejected": -94.39766693115234,
      "loss": 0.6506,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": -0.14505597949028015,
      "rewards/margins": 0.1382911652326584,
      "rewards/rejected": -0.28334707021713257,
      "step": 150
    },
    {
      "epoch": 0.1674515960230246,
      "grad_norm": 13.04813612493708,
      "learning_rate": 4.931828996974498e-07,
      "logits/chosen": -2.0861306190490723,
      "logits/rejected": -2.0900378227233887,
      "logps/chosen": -87.21963500976562,
      "logps/rejected": -114.88661193847656,
      "loss": 0.6528,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.13326936960220337,
      "rewards/margins": 0.1332504153251648,
      "rewards/rejected": -0.26651981472969055,
      "step": 160
    },
    {
      "epoch": 0.17791732077446362,
      "grad_norm": 14.778101043613702,
      "learning_rate": 4.909001458367866e-07,
      "logits/chosen": -1.9173187017440796,
      "logits/rejected": -1.8886661529541016,
      "logps/chosen": -81.811767578125,
      "logps/rejected": -101.0234375,
      "loss": 0.634,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.16403724253177643,
      "rewards/margins": 0.19587047398090363,
      "rewards/rejected": -0.35990768671035767,
      "step": 170
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 26.424629291070275,
      "learning_rate": 4.882952093833627e-07,
      "logits/chosen": -1.3328689336776733,
      "logits/rejected": -1.3415606021881104,
      "logps/chosen": -121.7206039428711,
      "logps/rejected": -163.23861694335938,
      "loss": 0.6215,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4643549919128418,
      "rewards/margins": 0.2982317805290222,
      "rewards/rejected": -0.7625867128372192,
      "step": 180
    },
    {
      "epoch": 0.1988487702773417,
      "grad_norm": 18.286232555939137,
      "learning_rate": 4.853715742087946e-07,
      "logits/chosen": -1.299851655960083,
      "logits/rejected": -1.2263530492782593,
      "logps/chosen": -130.7896270751953,
      "logps/rejected": -153.83375549316406,
      "loss": 0.6187,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": -0.5220120549201965,
      "rewards/margins": 0.28233885765075684,
      "rewards/rejected": -0.8043509721755981,
      "step": 190
    },
    {
      "epoch": 0.20931449502878074,
      "grad_norm": 22.532611448739736,
      "learning_rate": 4.821331504159906e-07,
      "logits/chosen": -1.2297742366790771,
      "logits/rejected": -1.1969741582870483,
      "logps/chosen": -117.6633071899414,
      "logps/rejected": -130.67193603515625,
      "loss": 0.6488,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.3429523706436157,
      "rewards/margins": 0.20222480595111847,
      "rewards/rejected": -0.545177161693573,
      "step": 200
    },
    {
      "epoch": 0.20931449502878074,
      "eval_logits/chosen": -1.2583706378936768,
      "eval_logits/rejected": -1.23958420753479,
      "eval_logps/chosen": -102.57432556152344,
      "eval_logps/rejected": -130.3724822998047,
      "eval_loss": 0.634810745716095,
      "eval_rewards/accuracies": 0.3373015820980072,
      "eval_rewards/chosen": -0.2808309495449066,
      "eval_rewards/margins": 0.21161039173603058,
      "eval_rewards/rejected": -0.492441326379776,
      "eval_runtime": 113.6607,
      "eval_samples_per_second": 17.596,
      "eval_steps_per_second": 0.554,
      "step": 200
    },
    {
      "epoch": 0.21978021978021978,
      "grad_norm": 43.96668983794973,
      "learning_rate": 4.785842691097342e-07,
      "logits/chosen": -1.166576623916626,
      "logits/rejected": -1.0878881216049194,
      "logps/chosen": -102.22654724121094,
      "logps/rejected": -119.87858581542969,
      "loss": 0.6449,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.30706560611724854,
      "rewards/margins": 0.2078002393245697,
      "rewards/rejected": -0.5148658752441406,
      "step": 210
    },
    {
      "epoch": 0.2302459445316588,
      "grad_norm": 21.60611363424848,
      "learning_rate": 4.7472967660421603e-07,
      "logits/chosen": -1.4838167428970337,
      "logits/rejected": -1.293268084526062,
      "logps/chosen": -140.9451446533203,
      "logps/rejected": -159.35256958007812,
      "loss": 0.6317,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.40442484617233276,
      "rewards/margins": 0.2767654359340668,
      "rewards/rejected": -0.6811902523040771,
      "step": 220
    },
    {
      "epoch": 0.24071166928309787,
      "grad_norm": 23.269156895827596,
      "learning_rate": 4.705745280752585e-07,
      "logits/chosen": -1.360938310623169,
      "logits/rejected": -1.2307772636413574,
      "logps/chosen": -92.72604370117188,
      "logps/rejected": -110.38468933105469,
      "loss": 0.6411,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.3325788080692291,
      "rewards/margins": 0.19615355134010315,
      "rewards/rejected": -0.5287323594093323,
      "step": 230
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 32.70987860826756,
      "learning_rate": 4.6612438066572555e-07,
      "logits/chosen": -1.1462054252624512,
      "logits/rejected": -0.9243119359016418,
      "logps/chosen": -129.86032104492188,
      "logps/rejected": -173.3503875732422,
      "loss": 0.6198,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.4205262064933777,
      "rewards/margins": 0.42327141761779785,
      "rewards/rejected": -0.8437975645065308,
      "step": 240
    },
    {
      "epoch": 0.2616431187859759,
      "grad_norm": 18.56295517177359,
      "learning_rate": 4.6138518605333664e-07,
      "logits/chosen": -0.9204837083816528,
      "logits/rejected": -0.8685296177864075,
      "logps/chosen": -83.56452178955078,
      "logps/rejected": -112.06929779052734,
      "loss": 0.6227,
      "rewards/accuracies": 0.24375000596046448,
      "rewards/chosen": -0.30188173055648804,
      "rewards/margins": 0.20583298802375793,
      "rewards/rejected": -0.5077147483825684,
      "step": 250
    },
    {
      "epoch": 0.272108843537415,
      "grad_norm": 24.278208900563758,
      "learning_rate": 4.5636328249082514e-07,
      "logits/chosen": -0.5377733111381531,
      "logits/rejected": -0.28726479411125183,
      "logps/chosen": -126.16239166259766,
      "logps/rejected": -144.44386291503906,
      "loss": 0.6132,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.43910473585128784,
      "rewards/margins": 0.27166199684143066,
      "rewards/rejected": -0.7107667922973633,
      "step": 260
    },
    {
      "epoch": 0.282574568288854,
      "grad_norm": 33.830045643602666,
      "learning_rate": 4.510653863290871e-07,
      "logits/chosen": -0.43365517258644104,
      "logits/rejected": -0.23527593910694122,
      "logps/chosen": -127.06233215332031,
      "logps/rejected": -150.31356811523438,
      "loss": 0.6191,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.4495798647403717,
      "rewards/margins": 0.3023374378681183,
      "rewards/rejected": -0.7519172430038452,
      "step": 270
    },
    {
      "epoch": 0.29304029304029305,
      "grad_norm": 24.405765366439173,
      "learning_rate": 4.4549858303465737e-07,
      "logits/chosen": -0.9159714579582214,
      "logits/rejected": -0.717955470085144,
      "logps/chosen": -109.26994323730469,
      "logps/rejected": -139.60409545898438,
      "loss": 0.622,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3430403769016266,
      "rewards/margins": 0.24182644486427307,
      "rewards/rejected": -0.5848668217658997,
      "step": 280
    },
    {
      "epoch": 0.3035060177917321,
      "grad_norm": 18.88355766247808,
      "learning_rate": 4.396703177135261e-07,
      "logits/chosen": -1.3694268465042114,
      "logits/rejected": -1.1670420169830322,
      "logps/chosen": -97.29615783691406,
      "logps/rejected": -123.79087829589844,
      "loss": 0.6219,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.3448092043399811,
      "rewards/margins": 0.27321183681488037,
      "rewards/rejected": -0.6180210709571838,
      "step": 290
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 24.161262620892497,
      "learning_rate": 4.335883851539693e-07,
      "logits/chosen": -1.5254216194152832,
      "logits/rejected": -1.4045015573501587,
      "logps/chosen": -147.19302368164062,
      "logps/rejected": -174.0820770263672,
      "loss": 0.6331,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.5355431437492371,
      "rewards/margins": 0.2548714876174927,
      "rewards/rejected": -0.7904146313667297,
      "step": 300
    },
    {
      "epoch": 0.3139717425431711,
      "eval_logits/chosen": -1.1872740983963013,
      "eval_logits/rejected": -1.0319762229919434,
      "eval_logps/chosen": -120.13069915771484,
      "eval_logps/rejected": -157.09765625,
      "eval_loss": 0.6194990277290344,
      "eval_rewards/accuracies": 0.3452380895614624,
      "eval_rewards/chosen": -0.45639467239379883,
      "eval_rewards/margins": 0.3032984435558319,
      "eval_rewards/rejected": -0.7596930265426636,
      "eval_runtime": 113.7203,
      "eval_samples_per_second": 17.587,
      "eval_steps_per_second": 0.554,
      "step": 300
    },
    {
      "epoch": 0.32443746729461015,
      "grad_norm": 27.882854283662915,
      "learning_rate": 4.272609194017105e-07,
      "logits/chosen": -0.8721631765365601,
      "logits/rejected": -0.47974568605422974,
      "logps/chosen": -142.952392578125,
      "logps/rejected": -164.13180541992188,
      "loss": 0.6108,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.44492608308792114,
      "rewards/margins": 0.36958834528923035,
      "rewards/rejected": -0.8145144581794739,
      "step": 310
    },
    {
      "epoch": 0.3349031920460492,
      "grad_norm": 40.79438243043005,
      "learning_rate": 4.2069638288135547e-07,
      "logits/chosen": 0.030854111537337303,
      "logits/rejected": 0.30916082859039307,
      "logps/chosen": -143.35768127441406,
      "logps/rejected": -215.4879608154297,
      "loss": 0.646,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.6657307147979736,
      "rewards/margins": 0.5068569779396057,
      "rewards/rejected": -1.1725876331329346,
      "step": 320
    },
    {
      "epoch": 0.3453689167974882,
      "grad_norm": 28.74932802994849,
      "learning_rate": 4.139035550786494e-07,
      "logits/chosen": 0.10449258983135223,
      "logits/rejected": 0.19263358414173126,
      "logps/chosen": -125.71956634521484,
      "logps/rejected": -157.4947509765625,
      "loss": 0.6184,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.5608196258544922,
      "rewards/margins": 0.2601728141307831,
      "rewards/rejected": -0.8209924697875977,
      "step": 330
    },
    {
      "epoch": 0.35583464154892724,
      "grad_norm": 20.26835641704999,
      "learning_rate": 4.0689152079869306e-07,
      "logits/chosen": -0.6976083517074585,
      "logits/rejected": -0.4943923354148865,
      "logps/chosen": -127.49371337890625,
      "logps/rejected": -158.2890625,
      "loss": 0.6271,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.5544435977935791,
      "rewards/margins": 0.3053310215473175,
      "rewards/rejected": -0.8597745895385742,
      "step": 340
    },
    {
      "epoch": 0.3663003663003663,
      "grad_norm": 27.63474707817879,
      "learning_rate": 3.99669658015821e-07,
      "logits/chosen": -0.6875920295715332,
      "logits/rejected": -0.5427245497703552,
      "logps/chosen": -149.16458129882812,
      "logps/rejected": -175.6387481689453,
      "loss": 0.6043,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.6470784544944763,
      "rewards/margins": 0.2719033360481262,
      "rewards/rejected": -0.9189817309379578,
      "step": 350
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 30.4860863672758,
      "learning_rate": 3.92247625331392e-07,
      "logits/chosen": 0.019889334216713905,
      "logits/rejected": 0.449519544839859,
      "logps/chosen": -158.36898803710938,
      "logps/rejected": -190.15274047851562,
      "loss": 0.5979,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.6107655763626099,
      "rewards/margins": 0.40659332275390625,
      "rewards/rejected": -1.0173588991165161,
      "step": 360
    },
    {
      "epoch": 0.3872318158032444,
      "grad_norm": 28.105949004143817,
      "learning_rate": 3.846353490562664e-07,
      "logits/chosen": 0.23578917980194092,
      "logits/rejected": 0.578147292137146,
      "logps/chosen": -144.15545654296875,
      "logps/rejected": -183.06863403320312,
      "loss": 0.6068,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.577487587928772,
      "rewards/margins": 0.37577182054519653,
      "rewards/rejected": -0.9532594680786133,
      "step": 370
    },
    {
      "epoch": 0.3976975405546834,
      "grad_norm": 19.345534639045226,
      "learning_rate": 3.768430099352445e-07,
      "logits/chosen": -0.3674705922603607,
      "logits/rejected": 0.07584401965141296,
      "logps/chosen": -130.8544464111328,
      "logps/rejected": -168.41697692871094,
      "loss": 0.6092,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": -0.49021464586257935,
      "rewards/margins": 0.39871546626091003,
      "rewards/rejected": -0.8889301419258118,
      "step": 380
    },
    {
      "epoch": 0.40816326530612246,
      "grad_norm": 29.65852362420373,
      "learning_rate": 3.6888102953122304e-07,
      "logits/chosen": -0.23846562206745148,
      "logits/rejected": -0.008897816762328148,
      "logps/chosen": -138.2523956298828,
      "logps/rejected": -194.8299102783203,
      "loss": 0.6241,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.5844453573226929,
      "rewards/margins": 0.4089936316013336,
      "rewards/rejected": -0.9934390187263489,
      "step": 390
    },
    {
      "epoch": 0.4186289900575615,
      "grad_norm": 42.485008339554724,
      "learning_rate": 3.607600562872785e-07,
      "logits/chosen": 0.0847388356924057,
      "logits/rejected": 0.3739756643772125,
      "logps/chosen": -133.99862670898438,
      "logps/rejected": -157.2317352294922,
      "loss": 0.6321,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.6070858836174011,
      "rewards/margins": 0.2605039179325104,
      "rewards/rejected": -0.8675897717475891,
      "step": 400
    },
    {
      "epoch": 0.4186289900575615,
      "eval_logits/chosen": 0.03354182466864586,
      "eval_logits/rejected": 0.37280067801475525,
      "eval_logps/chosen": -146.9637451171875,
      "eval_logps/rejected": -190.27566528320312,
      "eval_loss": 0.6099374294281006,
      "eval_rewards/accuracies": 0.363095223903656,
      "eval_rewards/chosen": -0.7247251272201538,
      "eval_rewards/margins": 0.36674803495407104,
      "eval_rewards/rejected": -1.0914732217788696,
      "eval_runtime": 113.6653,
      "eval_samples_per_second": 17.596,
      "eval_steps_per_second": 0.554,
      "step": 400
    },
    {
      "epoch": 0.4290947148090005,
      "grad_norm": 17.771390083818016,
      "learning_rate": 3.5249095128531856e-07,
      "logits/chosen": -0.3344365656375885,
      "logits/rejected": -0.13601410388946533,
      "logps/chosen": -168.97885131835938,
      "logps/rejected": -211.15029907226562,
      "loss": 0.6274,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.8895459175109863,
      "rewards/margins": 0.2965359091758728,
      "rewards/rejected": -1.186081886291504,
      "step": 410
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 29.30170184710936,
      "learning_rate": 3.4408477372034736e-07,
      "logits/chosen": -0.6253395080566406,
      "logits/rejected": -0.4562221169471741,
      "logps/chosen": -117.9905014038086,
      "logps/rejected": -139.88853454589844,
      "loss": 0.6305,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": -0.49884548783302307,
      "rewards/margins": 0.21792730689048767,
      "rewards/rejected": -0.7167727947235107,
      "step": 420
    },
    {
      "epoch": 0.4500261643118786,
      "grad_norm": 19.52934749077977,
      "learning_rate": 3.3555276610977276e-07,
      "logits/chosen": 0.060841239988803864,
      "logits/rejected": 0.28291866183280945,
      "logps/chosen": -119.32076263427734,
      "logps/rejected": -161.02288818359375,
      "loss": 0.6071,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.5378307104110718,
      "rewards/margins": 0.3328271806240082,
      "rewards/rejected": -0.8706579208374023,
      "step": 430
    },
    {
      "epoch": 0.4604918890633176,
      "grad_norm": 28.977327104522658,
      "learning_rate": 3.269063392575352e-07,
      "logits/chosen": 0.562627911567688,
      "logits/rejected": 0.706725001335144,
      "logps/chosen": -145.38662719726562,
      "logps/rejected": -177.89791870117188,
      "loss": 0.6155,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.7169743776321411,
      "rewards/margins": 0.19164128601551056,
      "rewards/rejected": -0.9086155891418457,
      "step": 440
    },
    {
      "epoch": 0.47095761381475665,
      "grad_norm": 21.22082345177454,
      "learning_rate": 3.1815705699316964e-07,
      "logits/chosen": 0.6601327657699585,
      "logits/rejected": 0.9368169903755188,
      "logps/chosen": -159.88311767578125,
      "logps/rejected": -201.655517578125,
      "loss": 0.6175,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.6956228017807007,
      "rewards/margins": 0.4413565993309021,
      "rewards/rejected": -1.1369794607162476,
      "step": 450
    },
    {
      "epoch": 0.48142333856619574,
      "grad_norm": 19.35642436173428,
      "learning_rate": 3.0931662070620794e-07,
      "logits/chosen": 0.47756925225257874,
      "logits/rejected": 1.0075833797454834,
      "logps/chosen": -136.54437255859375,
      "logps/rejected": -176.9443359375,
      "loss": 0.6108,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": -0.6124375462532043,
      "rewards/margins": 0.4467083811759949,
      "rewards/rejected": -1.0591459274291992,
      "step": 460
    },
    {
      "epoch": 0.49188906331763477,
      "grad_norm": 35.6743715566195,
      "learning_rate": 3.003968536966078e-07,
      "logits/chosen": 0.9841750264167786,
      "logits/rejected": 1.1566669940948486,
      "logps/chosen": -134.76565551757812,
      "logps/rejected": -183.22018432617188,
      "loss": 0.6033,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.6426252722740173,
      "rewards/margins": 0.34931105375289917,
      "rewards/rejected": -0.9919363856315613,
      "step": 470
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 31.355461154744457,
      "learning_rate": 2.9140968536213693e-07,
      "logits/chosen": 1.8672128915786743,
      "logits/rejected": 2.3499321937561035,
      "logps/chosen": -142.2679443359375,
      "logps/rejected": -197.79867553710938,
      "loss": 0.6029,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.6666483879089355,
      "rewards/margins": 0.5482696890830994,
      "rewards/rejected": -1.2149180173873901,
      "step": 480
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 39.792642390254535,
      "learning_rate": 2.823671352438608e-07,
      "logits/chosen": 2.002504825592041,
      "logits/rejected": 2.7407174110412598,
      "logps/chosen": -147.71644592285156,
      "logps/rejected": -184.10256958007812,
      "loss": 0.6191,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.5607318878173828,
      "rewards/margins": 0.49417656660079956,
      "rewards/rejected": -1.0549085140228271,
      "step": 490
    },
    {
      "epoch": 0.5232862375719518,
      "grad_norm": 19.501719693409513,
      "learning_rate": 2.73281296951072e-07,
      "logits/chosen": 2.3184399604797363,
      "logits/rejected": 2.6984035968780518,
      "logps/chosen": -179.01693725585938,
      "logps/rejected": -221.22781372070312,
      "loss": 0.6318,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.8433685302734375,
      "rewards/margins": 0.404899924993515,
      "rewards/rejected": -1.248268485069275,
      "step": 500
    },
    {
      "epoch": 0.5232862375719518,
      "eval_logits/chosen": 2.6547250747680664,
      "eval_logits/rejected": 2.9545063972473145,
      "eval_logps/chosen": -155.49295043945312,
      "eval_logps/rejected": -204.6371307373047,
      "eval_loss": 0.6104578375816345,
      "eval_rewards/accuracies": 0.3551587164402008,
      "eval_rewards/chosen": -0.8100170493125916,
      "eval_rewards/margins": 0.4250708818435669,
      "eval_rewards/rejected": -1.2350879907608032,
      "eval_runtime": 113.5938,
      "eval_samples_per_second": 17.607,
      "eval_steps_per_second": 0.555,
      "step": 500
    },
    {
      "epoch": 0.533751962323391,
      "grad_norm": 23.676894445603228,
      "learning_rate": 2.641643219871597e-07,
      "logits/chosen": 2.713271379470825,
      "logits/rejected": 2.843205213546753,
      "logps/chosen": -130.48731994628906,
      "logps/rejected": -174.28225708007812,
      "loss": 0.6232,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": -0.6893380284309387,
      "rewards/margins": 0.37240949273109436,
      "rewards/rejected": -1.061747431755066,
      "step": 510
    },
    {
      "epoch": 0.54421768707483,
      "grad_norm": 35.84824398777263,
      "learning_rate": 2.550284034980507e-07,
      "logits/chosen": 2.0955018997192383,
      "logits/rejected": 2.464780330657959,
      "logps/chosen": -167.99371337890625,
      "logps/rejected": -198.072021484375,
      "loss": 0.6346,
      "rewards/accuracies": 0.26875001192092896,
      "rewards/chosen": -0.9579731822013855,
      "rewards/margins": 0.2499997913837433,
      "rewards/rejected": -1.2079728841781616,
      "step": 520
    },
    {
      "epoch": 0.554683411826269,
      "grad_norm": 19.326207651996775,
      "learning_rate": 2.4588575996495794e-07,
      "logits/chosen": 1.578254222869873,
      "logits/rejected": 1.7954685688018799,
      "logps/chosen": -124.0101547241211,
      "logps/rejected": -157.9552764892578,
      "loss": 0.6063,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": -0.6386234760284424,
      "rewards/margins": 0.3164999485015869,
      "rewards/rejected": -0.9551234245300293,
      "step": 530
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 25.47159513000541,
      "learning_rate": 2.367486188632446e-07,
      "logits/chosen": 1.5957086086273193,
      "logits/rejected": 1.9952272176742554,
      "logps/chosen": -136.2624053955078,
      "logps/rejected": -160.97589111328125,
      "loss": 0.6202,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": -0.6916071772575378,
      "rewards/margins": 0.2573884129524231,
      "rewards/rejected": -0.9489954710006714,
      "step": 540
    },
    {
      "epoch": 0.5756148613291471,
      "grad_norm": 27.189003326527832,
      "learning_rate": 2.276292003092593e-07,
      "logits/chosen": 0.5969494581222534,
      "logits/rejected": 1.1224400997161865,
      "logps/chosen": -165.4878387451172,
      "logps/rejected": -201.8157958984375,
      "loss": 0.6072,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.7528584599494934,
      "rewards/margins": 0.4292448163032532,
      "rewards/rejected": -1.1821032762527466,
      "step": 550
    },
    {
      "epoch": 0.5860805860805861,
      "grad_norm": 19.01493291730421,
      "learning_rate": 2.185397007170141e-07,
      "logits/chosen": 0.5469252467155457,
      "logits/rejected": 0.9992968440055847,
      "logps/chosen": -109.37480163574219,
      "logps/rejected": -158.11180114746094,
      "loss": 0.607,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.5237552523612976,
      "rewards/margins": 0.4429488778114319,
      "rewards/rejected": -0.9667040705680847,
      "step": 560
    },
    {
      "epoch": 0.5965463108320251,
      "grad_norm": 34.53884646430518,
      "learning_rate": 2.094922764865619e-07,
      "logits/chosen": 0.47024235129356384,
      "logits/rejected": 1.3419710397720337,
      "logps/chosen": -152.4022674560547,
      "logps/rejected": -181.06829833984375,
      "loss": 0.6036,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.6253000497817993,
      "rewards/margins": 0.45260563492774963,
      "rewards/rejected": -1.0779056549072266,
      "step": 570
    },
    {
      "epoch": 0.6070120355834642,
      "grad_norm": 23.195476547368756,
      "learning_rate": 2.0049902774588797e-07,
      "logits/chosen": 0.7025114297866821,
      "logits/rejected": 1.4946035146713257,
      "logps/chosen": -155.9736328125,
      "logps/rejected": -197.6754150390625,
      "loss": 0.6174,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.7866873741149902,
      "rewards/margins": 0.5116966366767883,
      "rewards/rejected": -1.2983839511871338,
      "step": 580
    },
    {
      "epoch": 0.6174777603349032,
      "grad_norm": 28.28948495055075,
      "learning_rate": 1.9157198216806238e-07,
      "logits/chosen": 1.2547266483306885,
      "logits/rejected": 1.6798299551010132,
      "logps/chosen": -143.81948852539062,
      "logps/rejected": -180.07110595703125,
      "loss": 0.5974,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.7290927171707153,
      "rewards/margins": 0.344825804233551,
      "rewards/rejected": -1.0739185810089111,
      "step": 590
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 26.475084176869974,
      "learning_rate": 1.8272307888529274e-07,
      "logits/chosen": 1.0473191738128662,
      "logits/rejected": 1.4664316177368164,
      "logps/chosen": -151.5343017578125,
      "logps/rejected": -203.01600646972656,
      "loss": 0.5978,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.8304673433303833,
      "rewards/margins": 0.4478435516357422,
      "rewards/rejected": -1.278310775756836,
      "step": 600
    },
    {
      "epoch": 0.6279434850863422,
      "eval_logits/chosen": 0.9605558514595032,
      "eval_logits/rejected": 1.442029356956482,
      "eval_logps/chosen": -147.85601806640625,
      "eval_logps/rejected": -199.51206970214844,
      "eval_loss": 0.6014743447303772,
      "eval_rewards/accuracies": 0.3591269850730896,
      "eval_rewards/chosen": -0.733647882938385,
      "eval_rewards/margins": 0.4501895010471344,
      "eval_rewards/rejected": -1.1838374137878418,
      "eval_runtime": 113.6375,
      "eval_samples_per_second": 17.6,
      "eval_steps_per_second": 0.554,
      "step": 600
    },
    {
      "epoch": 0.6384092098377813,
      "grad_norm": 24.64809094353209,
      "learning_rate": 1.7396415252139288e-07,
      "logits/chosen": 0.9734107255935669,
      "logits/rejected": 1.4407756328582764,
      "logps/chosen": -136.4254150390625,
      "logps/rejected": -157.22183227539062,
      "loss": 0.6155,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": -0.6510840654373169,
      "rewards/margins": 0.3478087782859802,
      "rewards/rejected": -0.9988927841186523,
      "step": 610
    },
    {
      "epoch": 0.6488749345892203,
      "grad_norm": 24.74151564246123,
      "learning_rate": 1.6530691736402316e-07,
      "logits/chosen": 0.4937843680381775,
      "logits/rejected": 0.9153006672859192,
      "logps/chosen": -137.00144958496094,
      "logps/rejected": -177.6410675048828,
      "loss": 0.6073,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.725879967212677,
      "rewards/margins": 0.3728798031806946,
      "rewards/rejected": -1.0987598896026611,
      "step": 620
    },
    {
      "epoch": 0.6593406593406593,
      "grad_norm": 22.938989151746902,
      "learning_rate": 1.5676295169786864e-07,
      "logits/chosen": 0.19430339336395264,
      "logits/rejected": 0.6654868721961975,
      "logps/chosen": -176.47686767578125,
      "logps/rejected": -213.91622924804688,
      "loss": 0.5789,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.7090758085250854,
      "rewards/margins": 0.45301565527915955,
      "rewards/rejected": -1.1620914936065674,
      "step": 630
    },
    {
      "epoch": 0.6698063840920984,
      "grad_norm": 43.50900642344418,
      "learning_rate": 1.483436823197092e-07,
      "logits/chosen": 0.7957710027694702,
      "logits/rejected": 1.4320136308670044,
      "logps/chosen": -182.98187255859375,
      "logps/rejected": -233.4198455810547,
      "loss": 0.5792,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.872964084148407,
      "rewards/margins": 0.5010480284690857,
      "rewards/rejected": -1.3740123510360718,
      "step": 640
    },
    {
      "epoch": 0.6802721088435374,
      "grad_norm": 24.400296552508813,
      "learning_rate": 1.4006036925609243e-07,
      "logits/chosen": 1.008284330368042,
      "logits/rejected": 1.555418848991394,
      "logps/chosen": -128.50022888183594,
      "logps/rejected": -170.05349731445312,
      "loss": 0.59,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.5598582029342651,
      "rewards/margins": 0.4066081643104553,
      "rewards/rejected": -0.9664663076400757,
      "step": 650
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 31.773454484895552,
      "learning_rate": 1.319240907040458e-07,
      "logits/chosen": 0.5741680860519409,
      "logits/rejected": 1.095399022102356,
      "logps/chosen": -144.4388427734375,
      "logps/rejected": -190.87571716308594,
      "loss": 0.6023,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.5422704815864563,
      "rewards/margins": 0.48283880949020386,
      "rewards/rejected": -1.0251094102859497,
      "step": 660
    },
    {
      "epoch": 0.7012035583464155,
      "grad_norm": 37.13339246311252,
      "learning_rate": 1.239457282149695e-07,
      "logits/chosen": 0.8501984477043152,
      "logits/rejected": 1.152748942375183,
      "logps/chosen": -102.63143157958984,
      "logps/rejected": -154.28704833984375,
      "loss": 0.5949,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.4462759494781494,
      "rewards/margins": 0.41997307538986206,
      "rewards/rejected": -0.8662489652633667,
      "step": 670
    },
    {
      "epoch": 0.7116692830978545,
      "grad_norm": 32.56523005952964,
      "learning_rate": 1.1613595214152711e-07,
      "logits/chosen": 1.335402250289917,
      "logits/rejected": 1.811517357826233,
      "logps/chosen": -135.314453125,
      "logps/rejected": -198.50515747070312,
      "loss": 0.6071,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.587442934513092,
      "rewards/margins": 0.5670086145401001,
      "rewards/rejected": -1.154451608657837,
      "step": 680
    },
    {
      "epoch": 0.7221350078492935,
      "grad_norm": 23.014936124623496,
      "learning_rate": 1.0850520736699362e-07,
      "logits/chosen": 0.7806999683380127,
      "logits/rejected": 1.3070814609527588,
      "logps/chosen": -168.81216430664062,
      "logps/rejected": -202.5950469970703,
      "loss": 0.5659,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.7739877700805664,
      "rewards/margins": 0.40968823432922363,
      "rewards/rejected": -1.18367600440979,
      "step": 690
    },
    {
      "epoch": 0.7326007326007326,
      "grad_norm": 26.764323919821496,
      "learning_rate": 1.0106369933615042e-07,
      "logits/chosen": 0.9588180780410767,
      "logits/rejected": 1.4978833198547363,
      "logps/chosen": -165.7656707763672,
      "logps/rejected": -204.122802734375,
      "loss": 0.6113,
      "rewards/accuracies": 0.3687500059604645,
      "rewards/chosen": -0.9200434684753418,
      "rewards/margins": 0.37314558029174805,
      "rewards/rejected": -1.2931890487670898,
      "step": 700
    },
    {
      "epoch": 0.7326007326007326,
      "eval_logits/chosen": 1.1833491325378418,
      "eval_logits/rejected": 1.7187780141830444,
      "eval_logps/chosen": -150.68544006347656,
      "eval_logps/rejected": -204.91946411132812,
      "eval_loss": 0.5986347794532776,
      "eval_rewards/accuracies": 0.3650793731212616,
      "eval_rewards/chosen": -0.7619420289993286,
      "eval_rewards/margins": 0.4759688675403595,
      "eval_rewards/rejected": -1.2379108667373657,
      "eval_runtime": 113.6638,
      "eval_samples_per_second": 17.596,
      "eval_steps_per_second": 0.554,
      "step": 700
    },
    {
      "epoch": 0.7430664573521716,
      "grad_norm": 27.404707816356066,
      "learning_rate": 9.382138040640714e-08,
      "logits/chosen": 0.7452703714370728,
      "logits/rejected": 1.4386818408966064,
      "logps/chosen": -175.61923217773438,
      "logps/rejected": -214.57119750976562,
      "loss": 0.618,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.852981686592102,
      "rewards/margins": 0.44363918900489807,
      "rewards/rejected": -1.2966209650039673,
      "step": 710
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 27.03589930657382,
      "learning_rate": 8.678793653740632e-08,
      "logits/chosen": 1.3341294527053833,
      "logits/rejected": 1.880934476852417,
      "logps/chosen": -154.9960174560547,
      "logps/rejected": -193.92404174804688,
      "loss": 0.5852,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.8138143420219421,
      "rewards/margins": 0.4843239188194275,
      "rewards/rejected": -1.2981382608413696,
      "step": 720
    },
    {
      "epoch": 0.7639979068550498,
      "grad_norm": 19.503526282992237,
      "learning_rate": 7.997277433690983e-08,
      "logits/chosen": 1.2488057613372803,
      "logits/rejected": 1.543897271156311,
      "logps/chosen": -179.2456817626953,
      "logps/rejected": -218.6804656982422,
      "loss": 0.6017,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.9957185983657837,
      "rewards/margins": 0.3561645448207855,
      "rewards/rejected": -1.3518832921981812,
      "step": 730
    },
    {
      "epoch": 0.7744636316064888,
      "grad_norm": 24.561492093850955,
      "learning_rate": 7.338500848029602e-08,
      "logits/chosen": 0.7433587908744812,
      "logits/rejected": 1.235414981842041,
      "logps/chosen": -166.8399200439453,
      "logps/rejected": -215.26876831054688,
      "loss": 0.6178,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.9067522883415222,
      "rewards/margins": 0.41579413414001465,
      "rewards/rejected": -1.322546362876892,
      "step": 740
    },
    {
      "epoch": 0.7849293563579278,
      "grad_norm": 28.115986705653192,
      "learning_rate": 6.70334495204884e-08,
      "logits/chosen": 0.6296231150627136,
      "logits/rejected": 1.1273549795150757,
      "logps/chosen": -182.21527099609375,
      "logps/rejected": -222.38735961914062,
      "loss": 0.6104,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": -0.9979060292243958,
      "rewards/margins": 0.37623411417007446,
      "rewards/rejected": -1.3741401433944702,
      "step": 750
    },
    {
      "epoch": 0.7953950811093669,
      "grad_norm": 25.524078897067774,
      "learning_rate": 6.092659210462231e-08,
      "logits/chosen": 0.6932498216629028,
      "logits/rejected": 1.2560994625091553,
      "logps/chosen": -163.45713806152344,
      "logps/rejected": -210.75875854492188,
      "loss": 0.6428,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": -0.8338336944580078,
      "rewards/margins": 0.41811808943748474,
      "rewards/rejected": -1.2519516944885254,
      "step": 760
    },
    {
      "epoch": 0.8058608058608059,
      "grad_norm": 19.037858132036448,
      "learning_rate": 5.507260361320737e-08,
      "logits/chosen": 0.7545400261878967,
      "logits/rejected": 1.186693787574768,
      "logps/chosen": -139.1665802001953,
      "logps/rejected": -173.541748046875,
      "loss": 0.5694,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": -0.6477771401405334,
      "rewards/margins": 0.3577363193035126,
      "rewards/rejected": -1.0055135488510132,
      "step": 770
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 25.539683354526822,
      "learning_rate": 4.947931323697982e-08,
      "logits/chosen": 0.8985282778739929,
      "logits/rejected": 1.1507294178009033,
      "logps/chosen": -127.5772933959961,
      "logps/rejected": -150.71971130371094,
      "loss": 0.6274,
      "rewards/accuracies": 0.2562499940395355,
      "rewards/chosen": -0.5861515402793884,
      "rewards/margins": 0.2556864023208618,
      "rewards/rejected": -0.8418378829956055,
      "step": 780
    },
    {
      "epoch": 0.826792255363684,
      "grad_norm": 29.296410897786252,
      "learning_rate": 4.415420150605398e-08,
      "logits/chosen": 0.5066564083099365,
      "logits/rejected": 1.2835947275161743,
      "logps/chosen": -169.21194458007812,
      "logps/rejected": -226.9739990234375,
      "loss": 0.5972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.7719463109970093,
      "rewards/margins": 0.5745865702629089,
      "rewards/rejected": -1.346532940864563,
      "step": 790
    },
    {
      "epoch": 0.837257980115123,
      "grad_norm": 23.071826857167007,
      "learning_rate": 3.9104390285376374e-08,
      "logits/chosen": 0.2603650689125061,
      "logits/rejected": 0.9912735819816589,
      "logps/chosen": -189.1995086669922,
      "logps/rejected": -238.74069213867188,
      "loss": 0.5885,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.8696195483207703,
      "rewards/margins": 0.551138162612915,
      "rewards/rejected": -1.4207580089569092,
      "step": 800
    },
    {
      "epoch": 0.837257980115123,
      "eval_logits/chosen": 0.5612532496452332,
      "eval_logits/rejected": 1.0127543210983276,
      "eval_logps/chosen": -141.69253540039062,
      "eval_logps/rejected": -192.48452758789062,
      "eval_loss": 0.5973930954933167,
      "eval_rewards/accuracies": 0.369047611951828,
      "eval_rewards/chosen": -0.672012984752655,
      "eval_rewards/margins": 0.4415486454963684,
      "eval_rewards/rejected": -1.113561749458313,
      "eval_runtime": 113.6177,
      "eval_samples_per_second": 17.603,
      "eval_steps_per_second": 0.554,
      "step": 800
    },
    {
      "epoch": 0.847723704866562,
      "grad_norm": 17.313265289048484,
      "learning_rate": 3.433663324986208e-08,
      "logits/chosen": 0.3017066419124603,
      "logits/rejected": 0.7334527373313904,
      "logps/chosen": -151.9001007080078,
      "logps/rejected": -182.63177490234375,
      "loss": 0.613,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6817248463630676,
      "rewards/margins": 0.36351272463798523,
      "rewards/rejected": -1.0452375411987305,
      "step": 810
    },
    {
      "epoch": 0.858189429618001,
      "grad_norm": 32.324000689082936,
      "learning_rate": 2.9857306851953897e-08,
      "logits/chosen": 0.6850260496139526,
      "logits/rejected": 1.2093479633331299,
      "logps/chosen": -144.95767211914062,
      "logps/rejected": -198.352783203125,
      "loss": 0.6,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.6805930733680725,
      "rewards/margins": 0.44533902406692505,
      "rewards/rejected": -1.1259320974349976,
      "step": 820
    },
    {
      "epoch": 0.8686551543694401,
      "grad_norm": 27.72643919363414,
      "learning_rate": 2.567240179368185e-08,
      "logits/chosen": -0.07449465245008469,
      "logits/rejected": 0.24005027115345,
      "logps/chosen": -154.6387176513672,
      "logps/rejected": -227.32882690429688,
      "loss": 0.6169,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.7048233151435852,
      "rewards/margins": 0.4948086142539978,
      "rewards/rejected": -1.199631929397583,
      "step": 830
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 23.57031908849105,
      "learning_rate": 2.1787515014630357e-08,
      "logits/chosen": 0.2434501200914383,
      "logits/rejected": 0.7666997313499451,
      "logps/chosen": -151.32212829589844,
      "logps/rejected": -209.18057250976562,
      "loss": 0.5855,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.6667734980583191,
      "rewards/margins": 0.47117409110069275,
      "rewards/rejected": -1.1379475593566895,
      "step": 840
    },
    {
      "epoch": 0.8895866038723181,
      "grad_norm": 27.822093386580878,
      "learning_rate": 1.820784220652766e-08,
      "logits/chosen": 0.09173062443733215,
      "logits/rejected": 0.7935197949409485,
      "logps/chosen": -160.8597869873047,
      "logps/rejected": -215.7743377685547,
      "loss": 0.5858,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.6746786236763,
      "rewards/margins": 0.5673048496246338,
      "rewards/rejected": -1.2419836521148682,
      "step": 850
    },
    {
      "epoch": 0.9000523286237572,
      "grad_norm": 24.993719056225927,
      "learning_rate": 1.4938170864468636e-08,
      "logits/chosen": -0.022973239421844482,
      "logits/rejected": 0.728354811668396,
      "logps/chosen": -177.0865936279297,
      "logps/rejected": -222.30313110351562,
      "loss": 0.587,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.7254469990730286,
      "rewards/margins": 0.5123113989830017,
      "rewards/rejected": -1.2377583980560303,
      "step": 860
    },
    {
      "epoch": 0.9105180533751962,
      "grad_norm": 15.152551046729396,
      "learning_rate": 1.1982873884064465e-08,
      "logits/chosen": 0.3221861720085144,
      "logits/rejected": 0.7626439332962036,
      "logps/chosen": -138.0549774169922,
      "logps/rejected": -176.89402770996094,
      "loss": 0.6052,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": -0.6149898767471313,
      "rewards/margins": 0.41914796829223633,
      "rewards/rejected": -1.0341379642486572,
      "step": 870
    },
    {
      "epoch": 0.9209837781266352,
      "grad_norm": 34.8714524353856,
      "learning_rate": 9.345903713082304e-09,
      "logits/chosen": 0.6613011360168457,
      "logits/rejected": 0.8276697397232056,
      "logps/chosen": -124.2752914428711,
      "logps/rejected": -161.05874633789062,
      "loss": 0.6179,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": -0.602181077003479,
      "rewards/margins": 0.27649611234664917,
      "rewards/rejected": -0.8786771893501282,
      "step": 880
    },
    {
      "epoch": 0.9314495028780743,
      "grad_norm": 17.42009363611703,
      "learning_rate": 7.030787065396865e-09,
      "logits/chosen": 0.14433155953884125,
      "logits/rejected": 0.636074423789978,
      "logps/chosen": -144.4778289794922,
      "logps/rejected": -189.1996612548828,
      "loss": 0.6092,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.6060083508491516,
      "rewards/margins": 0.42767366766929626,
      "rewards/rejected": -1.033682107925415,
      "step": 890
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 25.137415279564916,
      "learning_rate": 5.04062020432286e-09,
      "logits/chosen": 0.5784530639648438,
      "logits/rejected": 1.0418832302093506,
      "logps/chosen": -118.29461669921875,
      "logps/rejected": -145.42251586914062,
      "loss": 0.595,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.4733172357082367,
      "rewards/margins": 0.31973880529403687,
      "rewards/rejected": -0.793056070804596,
      "step": 900
    },
    {
      "epoch": 0.9419152276295133,
      "eval_logits/chosen": 0.4325558543205261,
      "eval_logits/rejected": 0.9106192588806152,
      "eval_logps/chosen": -136.28819274902344,
      "eval_logps/rejected": -189.55056762695312,
      "eval_loss": 0.5958317518234253,
      "eval_rewards/accuracies": 0.3710317313671112,
      "eval_rewards/chosen": -0.6179695725440979,
      "eval_rewards/margins": 0.4662524461746216,
      "eval_rewards/rejected": -1.0842220783233643,
      "eval_runtime": 113.6384,
      "eval_samples_per_second": 17.6,
      "eval_steps_per_second": 0.554,
      "step": 900
    },
    {
      "epoch": 0.9523809523809523,
      "grad_norm": 48.333955356090875,
      "learning_rate": 3.3780648016376866e-09,
      "logits/chosen": -0.08735128492116928,
      "logits/rejected": 0.43545690178871155,
      "logps/chosen": -137.79881286621094,
      "logps/rejected": -196.9468231201172,
      "loss": 0.593,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.7070980072021484,
      "rewards/margins": 0.4561308026313782,
      "rewards/rejected": -1.1632287502288818,
      "step": 910
    },
    {
      "epoch": 0.9628466771323915,
      "grad_norm": 28.43671889890296,
      "learning_rate": 2.0453443778310766e-09,
      "logits/chosen": -0.048371605575084686,
      "logits/rejected": 0.41192588210105896,
      "logps/chosen": -149.82281494140625,
      "logps/rejected": -189.8140411376953,
      "loss": 0.6088,
      "rewards/accuracies": 0.3687500059604645,
      "rewards/chosen": -0.6467713713645935,
      "rewards/margins": 0.3985586166381836,
      "rewards/rejected": -1.0453299283981323,
      "step": 920
    },
    {
      "epoch": 0.9733124018838305,
      "grad_norm": 37.55350929589785,
      "learning_rate": 1.0442413283435758e-09,
      "logits/chosen": -0.05256899446249008,
      "logits/rejected": 0.6733183860778809,
      "logps/chosen": -138.57061767578125,
      "logps/rejected": -181.3246307373047,
      "loss": 0.5951,
      "rewards/accuracies": 0.3687500059604645,
      "rewards/chosen": -0.5108522772789001,
      "rewards/margins": 0.48942360281944275,
      "rewards/rejected": -1.0002758502960205,
      "step": 930
    },
    {
      "epoch": 0.9837781266352695,
      "grad_norm": 23.109646236441566,
      "learning_rate": 3.760945397705828e-10,
      "logits/chosen": 0.17422077059745789,
      "logits/rejected": 0.798999547958374,
      "logps/chosen": -132.8301239013672,
      "logps/rejected": -187.94265747070312,
      "loss": 0.5961,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.599638819694519,
      "rewards/margins": 0.4609258770942688,
      "rewards/rejected": -1.060564637184143,
      "step": 940
    },
    {
      "epoch": 0.9942438513867086,
      "grad_norm": 18.08118855716985,
      "learning_rate": 4.17975992204056e-11,
      "logits/chosen": 0.5573434829711914,
      "logits/rejected": 0.9020156860351562,
      "logps/chosen": -134.12124633789062,
      "logps/rejected": -163.89334106445312,
      "loss": 0.618,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.6402407288551331,
      "rewards/margins": 0.31644412875175476,
      "rewards/rejected": -0.9566848874092102,
      "step": 950
    },
    {
      "epoch": 0.9994767137624281,
      "step": 955,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 0.0165,
      "train_samples_per_second": 369998.418,
      "train_steps_per_second": 5810.543
    }
  ],
  "logging_steps": 10,
  "max_steps": 96,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}