{
  "best_metric": 1.3159173727035522,
  "best_model_checkpoint": "saves/Gemma-7B-It/lora/orpo-salt-half/checkpoint-1500",
  "epoch": 2.9974597798475866,
  "eval_steps": 500,
  "global_step": 1770,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01693480101608806,
      "grad_norm": 4.468957901000977,
      "learning_rate": 4.999614014035063e-06,
      "logits/chosen": 207.0987548828125,
      "logits/rejected": 208.2423858642578,
      "logps/chosen": -2.50309681892395,
      "logps/rejected": -2.976195812225342,
      "loss": 2.5989,
      "odds_ratio_loss": 0.9582594633102417,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.25030970573425293,
      "rewards/margins": 0.04730992019176483,
      "rewards/rejected": -0.29761961102485657,
      "sft_loss": 2.50309681892395,
      "step": 10
    },
    {
      "epoch": 0.03386960203217612,
      "grad_norm": 3.255079984664917,
      "learning_rate": 4.998440543386042e-06,
      "logits/chosen": 207.7864227294922,
      "logits/rejected": 210.0211944580078,
      "logps/chosen": -2.684905529022217,
      "logps/rejected": -2.9186933040618896,
      "loss": 2.776,
      "odds_ratio_loss": 0.9109451174736023,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.26849058270454407,
      "rewards/margins": 0.023378772661089897,
      "rewards/rejected": -0.2918693423271179,
      "sft_loss": 2.684905529022217,
      "step": 20
    },
    {
      "epoch": 0.05080440304826418,
      "grad_norm": 2.5176427364349365,
      "learning_rate": 4.996479918381253e-06,
      "logits/chosen": 212.6219024658203,
      "logits/rejected": 213.315673828125,
      "logps/chosen": -2.225409746170044,
      "logps/rejected": -2.3403306007385254,
      "loss": 2.3097,
      "odds_ratio_loss": 0.8430029153823853,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.22254100441932678,
      "rewards/margins": 0.01149209588766098,
      "rewards/rejected": -0.23403307795524597,
      "sft_loss": 2.225409746170044,
      "step": 30
    },
    {
      "epoch": 0.06773920406435224,
      "grad_norm": 2.6412463188171387,
      "learning_rate": 4.993732756731818e-06,
      "logits/chosen": 212.7200164794922,
      "logits/rejected": 213.7319793701172,
      "logps/chosen": -2.2491908073425293,
      "logps/rejected": -2.6265273094177246,
      "loss": 2.3215,
      "odds_ratio_loss": 0.7235576510429382,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.22491908073425293,
      "rewards/margins": 0.03773364797234535,
      "rewards/rejected": -0.262652724981308,
      "sft_loss": 2.2491908073425293,
      "step": 40
    },
    {
      "epoch": 0.0846740050804403,
      "grad_norm": 4.878486633300781,
      "learning_rate": 4.9901999239537345e-06,
      "logits/chosen": 218.13858032226562,
      "logits/rejected": 218.47933959960938,
      "logps/chosen": -2.325134754180908,
      "logps/rejected": -2.5971357822418213,
      "loss": 2.4052,
      "odds_ratio_loss": 0.8009692430496216,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.23251347243785858,
      "rewards/margins": 0.027200128883123398,
      "rewards/rejected": -0.2597135901451111,
      "sft_loss": 2.325134754180908,
      "step": 50
    },
    {
      "epoch": 0.10160880609652836,
      "grad_norm": 1.934574842453003,
      "learning_rate": 4.985882533095186e-06,
      "logits/chosen": 218.4595184326172,
      "logits/rejected": 218.9486083984375,
      "logps/chosen": -2.1272051334381104,
      "logps/rejected": -2.2577621936798096,
      "loss": 2.2131,
      "odds_ratio_loss": 0.8588129878044128,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.21272051334381104,
      "rewards/margins": 0.013055694289505482,
      "rewards/rejected": -0.22577619552612305,
      "sft_loss": 2.1272051334381104,
      "step": 60
    },
    {
      "epoch": 0.11854360711261643,
      "grad_norm": 8.395120620727539,
      "learning_rate": 4.9807819443858705e-06,
      "logits/chosen": 221.9539031982422,
      "logits/rejected": 223.3571319580078,
      "logps/chosen": -2.175661087036133,
      "logps/rejected": -2.3411145210266113,
      "loss": 2.2597,
      "odds_ratio_loss": 0.8399366140365601,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.2175661027431488,
      "rewards/margins": 0.016545329242944717,
      "rewards/rejected": -0.2341114580631256,
      "sft_loss": 2.175661087036133,
      "step": 70
    },
    {
      "epoch": 0.1354784081287045,
      "grad_norm": 4.525905132293701,
      "learning_rate": 4.9748997648084404e-06,
      "logits/chosen": 224.2382354736328,
      "logits/rejected": 224.88278198242188,
      "logps/chosen": -1.8507779836654663,
      "logps/rejected": -2.099276304244995,
      "loss": 1.9368,
      "odds_ratio_loss": 0.8602777719497681,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18507780134677887,
      "rewards/margins": 0.024849826470017433,
      "rewards/rejected": -0.20992763340473175,
      "sft_loss": 1.8507779836654663,
      "step": 80
    },
    {
      "epoch": 0.15241320914479256,
      "grad_norm": 4.903147220611572,
      "learning_rate": 4.96823784759222e-06,
      "logits/chosen": 225.79443359375,
      "logits/rejected": 226.6028289794922,
      "logps/chosen": -1.7729737758636475,
      "logps/rejected": -2.0627281665802,
      "loss": 1.848,
      "odds_ratio_loss": 0.7500249147415161,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.17729738354682922,
      "rewards/margins": 0.02897545136511326,
      "rewards/rejected": -0.20627284049987793,
      "sft_loss": 1.7729737758636475,
      "step": 90
    },
    {
      "epoch": 0.1693480101608806,
      "grad_norm": 3.5224053859710693,
      "learning_rate": 4.960798291629323e-06,
      "logits/chosen": 232.6717071533203,
      "logits/rejected": 232.73568725585938,
      "logps/chosen": -1.7854640483856201,
      "logps/rejected": -1.9538242816925049,
      "loss": 1.8613,
      "odds_ratio_loss": 0.7585908770561218,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.17854642868041992,
      "rewards/margins": 0.016835981979966164,
      "rewards/rejected": -0.19538240134716034,
      "sft_loss": 1.7854640483856201,
      "step": 100
    },
    {
      "epoch": 0.18628281117696868,
      "grad_norm": 6.120446681976318,
      "learning_rate": 4.952583440813383e-06,
      "logits/chosen": 235.1363983154297,
      "logits/rejected": 236.200927734375,
      "logps/chosen": -1.8061208724975586,
      "logps/rejected": -2.0079829692840576,
      "loss": 1.8823,
      "odds_ratio_loss": 0.7622145414352417,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.18061205744743347,
      "rewards/margins": 0.020186223089694977,
      "rewards/rejected": -0.20079830288887024,
      "sft_loss": 1.8061208724975586,
      "step": 110
    },
    {
      "epoch": 0.20321761219305673,
      "grad_norm": 1.8270832300186157,
      "learning_rate": 4.943595883301086e-06,
      "logits/chosen": 234.9897918701172,
      "logits/rejected": 235.3126983642578,
      "logps/chosen": -1.6784818172454834,
      "logps/rejected": -1.882794737815857,
      "loss": 1.7524,
      "odds_ratio_loss": 0.7392091155052185,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.16784818470478058,
      "rewards/margins": 0.02043129876255989,
      "rewards/rejected": -0.18827947974205017,
      "sft_loss": 1.6784818172454834,
      "step": 120
    },
    {
      "epoch": 0.2201524132091448,
      "grad_norm": 5.280871868133545,
      "learning_rate": 4.933838450696757e-06,
      "logits/chosen": 238.5899200439453,
      "logits/rejected": 238.400390625,
      "logps/chosen": -1.512404203414917,
      "logps/rejected": -1.6778627634048462,
      "loss": 1.5856,
      "odds_ratio_loss": 0.7318023443222046,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.15124042332172394,
      "rewards/margins": 0.0165458582341671,
      "rewards/rejected": -0.16778628528118134,
      "sft_loss": 1.512404203414917,
      "step": 130
    },
    {
      "epoch": 0.23708721422523285,
      "grad_norm": 4.924640655517578,
      "learning_rate": 4.923314217160234e-06,
      "logits/chosen": 240.5208282470703,
      "logits/rejected": 241.1940460205078,
      "logps/chosen": -1.6607558727264404,
      "logps/rejected": -1.8365033864974976,
      "loss": 1.7352,
      "odds_ratio_loss": 0.7449204921722412,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.16607561707496643,
      "rewards/margins": 0.017574718222022057,
      "rewards/rejected": -0.18365031480789185,
      "sft_loss": 1.6607558727264404,
      "step": 140
    },
    {
      "epoch": 0.2540220152413209,
      "grad_norm": 1.817272663116455,
      "learning_rate": 4.9120264984383285e-06,
      "logits/chosen": 242.286376953125,
      "logits/rejected": 242.5799560546875,
      "logps/chosen": -1.3862061500549316,
      "logps/rejected": -1.5650876760482788,
      "loss": 1.454,
      "odds_ratio_loss": 0.6781331300735474,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13862061500549316,
      "rewards/margins": 0.01788817159831524,
      "rewards/rejected": -0.15650877356529236,
      "sft_loss": 1.3862061500549316,
      "step": 150
    },
    {
      "epoch": 0.270956816257409,
      "grad_norm": 1.3204820156097412,
      "learning_rate": 4.899978850820176e-06,
      "logits/chosen": 244.1859588623047,
      "logits/rejected": 244.36865234375,
      "logps/chosen": -1.5278394222259521,
      "logps/rejected": -1.7709777355194092,
      "loss": 1.5938,
      "odds_ratio_loss": 0.6600898504257202,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.15278393030166626,
      "rewards/margins": 0.02431386150419712,
      "rewards/rejected": -0.17709779739379883,
      "sft_loss": 1.5278394222259521,
      "step": 160
    },
    {
      "epoch": 0.28789161727349705,
      "grad_norm": 2.3198039531707764,
      "learning_rate": 4.887175070016795e-06,
      "logits/chosen": 242.52523803710938,
      "logits/rejected": 243.25057983398438,
      "logps/chosen": -1.3887180089950562,
      "logps/rejected": -1.6768462657928467,
      "loss": 1.4533,
      "odds_ratio_loss": 0.6456118822097778,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13887180387973785,
      "rewards/margins": 0.02881282940506935,
      "rewards/rejected": -0.1676846444606781,
      "sft_loss": 1.3887180089950562,
      "step": 170
    },
    {
      "epoch": 0.3048264182895851,
      "grad_norm": 2.7539143562316895,
      "learning_rate": 4.873619189965217e-06,
      "logits/chosen": 246.211181640625,
      "logits/rejected": 246.160400390625,
      "logps/chosen": -1.5571125745773315,
      "logps/rejected": -1.8048861026763916,
      "loss": 1.6256,
      "odds_ratio_loss": 0.684849202632904,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.15571126341819763,
      "rewards/margins": 0.024777347221970558,
      "rewards/rejected": -0.18048861622810364,
      "sft_loss": 1.5571125745773315,
      "step": 180
    },
    {
      "epoch": 0.32176121930567314,
      "grad_norm": 1.5920647382736206,
      "learning_rate": 4.859315481557563e-06,
      "logits/chosen": 243.8936309814453,
      "logits/rejected": 244.6537628173828,
      "logps/chosen": -1.3444526195526123,
      "logps/rejected": -1.622938871383667,
      "loss": 1.4127,
      "odds_ratio_loss": 0.6820069551467896,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.13444526493549347,
      "rewards/margins": 0.027848612517118454,
      "rewards/rejected": -0.16229388117790222,
      "sft_loss": 1.3444526195526123,
      "step": 190
    },
    {
      "epoch": 0.3386960203217612,
      "grad_norm": 1.2194499969482422,
      "learning_rate": 4.84426845129546e-06,
      "logits/chosen": 245.6062774658203,
      "logits/rejected": 245.9265899658203,
      "logps/chosen": -1.4105961322784424,
      "logps/rejected": -1.591505765914917,
      "loss": 1.4778,
      "odds_ratio_loss": 0.6719616055488586,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.14105962216854095,
      "rewards/margins": 0.018090957775712013,
      "rewards/rejected": -0.15915057063102722,
      "sft_loss": 1.4105961322784424,
      "step": 200
    },
    {
      "epoch": 0.3556308213378493,
      "grad_norm": 1.6798571348190308,
      "learning_rate": 4.828482839870233e-06,
      "logits/chosen": 244.5519561767578,
      "logits/rejected": 244.9039764404297,
      "logps/chosen": -1.4486945867538452,
      "logps/rejected": -1.5620288848876953,
      "loss": 1.5235,
      "odds_ratio_loss": 0.7479228973388672,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.14486947655677795,
      "rewards/margins": 0.011333415284752846,
      "rewards/rejected": -0.15620288252830505,
      "sft_loss": 1.4486945867538452,
      "step": 210
    },
    {
      "epoch": 0.37256562235393736,
      "grad_norm": 1.2994054555892944,
      "learning_rate": 4.811963620669314e-06,
      "logits/chosen": 247.3550567626953,
      "logits/rejected": 247.1066436767578,
      "logps/chosen": -1.46903395652771,
      "logps/rejected": -1.5901343822479248,
      "loss": 1.5401,
      "odds_ratio_loss": 0.7104851007461548,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.14690342545509338,
      "rewards/margins": 0.012110031209886074,
      "rewards/rejected": -0.15901342034339905,
      "sft_loss": 1.46903395652771,
      "step": 220
    },
    {
      "epoch": 0.3895004233700254,
      "grad_norm": 1.8322347402572632,
      "learning_rate": 4.794715998209328e-06,
      "logits/chosen": 245.6450653076172,
      "logits/rejected": 246.178955078125,
      "logps/chosen": -1.3713748455047607,
      "logps/rejected": -1.5887004137039185,
      "loss": 1.4399,
      "odds_ratio_loss": 0.6854357719421387,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1371375024318695,
      "rewards/margins": 0.02173253521323204,
      "rewards/rejected": -0.15887005627155304,
      "sft_loss": 1.3713748455047607,
      "step": 230
    },
    {
      "epoch": 0.40643522438611346,
      "grad_norm": 1.3373929262161255,
      "learning_rate": 4.7767454064963724e-06,
      "logits/chosen": 246.9872283935547,
      "logits/rejected": 247.23184204101562,
      "logps/chosen": -1.418143391609192,
      "logps/rejected": -1.6108840703964233,
      "loss": 1.485,
      "odds_ratio_loss": 0.6687358617782593,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.14181432127952576,
      "rewards/margins": 0.019274089485406876,
      "rewards/rejected": -0.16108840703964233,
      "sft_loss": 1.418143391609192,
      "step": 240
    },
    {
      "epoch": 0.42337002540220153,
      "grad_norm": 2.7464914321899414,
      "learning_rate": 4.758057507313987e-06,
      "logits/chosen": 246.5187530517578,
      "logits/rejected": 247.2135772705078,
      "logps/chosen": -1.3453131914138794,
      "logps/rejected": -1.5121968984603882,
      "loss": 1.4146,
      "odds_ratio_loss": 0.6926182508468628,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13453131914138794,
      "rewards/margins": 0.0166883897036314,
      "rewards/rejected": -0.1512196958065033,
      "sft_loss": 1.3453131914138794,
      "step": 250
    },
    {
      "epoch": 0.4403048264182896,
      "grad_norm": 6.080187797546387,
      "learning_rate": 4.73865818843936e-06,
      "logits/chosen": 247.1885223388672,
      "logits/rejected": 247.33627319335938,
      "logps/chosen": -1.4415591955184937,
      "logps/rejected": -1.6857059001922607,
      "loss": 1.5104,
      "odds_ratio_loss": 0.6885947585105896,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14415593445301056,
      "rewards/margins": 0.0244146715849638,
      "rewards/rejected": -0.1685706079006195,
      "sft_loss": 1.4415591955184937,
      "step": 260
    },
    {
      "epoch": 0.4572396274343776,
      "grad_norm": 1.4364287853240967,
      "learning_rate": 4.718553561788339e-06,
      "logits/chosen": 247.25503540039062,
      "logits/rejected": 247.4564971923828,
      "logps/chosen": -1.3397562503814697,
      "logps/rejected": -1.5836654901504517,
      "loss": 1.4039,
      "odds_ratio_loss": 0.6409639120101929,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13397563993930817,
      "rewards/margins": 0.024390894919633865,
      "rewards/rejected": -0.15836653113365173,
      "sft_loss": 1.3397562503814697,
      "step": 270
    },
    {
      "epoch": 0.4741744284504657,
      "grad_norm": 1.979526162147522,
      "learning_rate": 4.697749961489822e-06,
      "logits/chosen": 246.6254425048828,
      "logits/rejected": 246.77536010742188,
      "logps/chosen": -1.5182511806488037,
      "logps/rejected": -1.7493031024932861,
      "loss": 1.5866,
      "odds_ratio_loss": 0.6837342381477356,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.15182514488697052,
      "rewards/margins": 0.023105164989829063,
      "rewards/rejected": -0.17493028938770294,
      "sft_loss": 1.5182511806488037,
      "step": 280
    },
    {
      "epoch": 0.4911092294665538,
      "grad_norm": 2.0010595321655273,
      "learning_rate": 4.67625394189013e-06,
      "logits/chosen": 246.59414672851562,
      "logits/rejected": 246.92410278320312,
      "logps/chosen": -1.317578673362732,
      "logps/rejected": -1.5447911024093628,
      "loss": 1.381,
      "odds_ratio_loss": 0.6341525316238403,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.13175788521766663,
      "rewards/margins": 0.022721242159605026,
      "rewards/rejected": -0.15447911620140076,
      "sft_loss": 1.317578673362732,
      "step": 290
    },
    {
      "epoch": 0.5080440304826418,
      "grad_norm": 3.1077401638031006,
      "learning_rate": 4.654072275488016e-06,
      "logits/chosen": 248.23709106445312,
      "logits/rejected": 248.05142211914062,
      "logps/chosen": -1.2916858196258545,
      "logps/rejected": -1.4983139038085938,
      "loss": 1.3555,
      "odds_ratio_loss": 0.6378514766693115,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1291685849428177,
      "rewards/margins": 0.020662816241383553,
      "rewards/rejected": -0.14983141422271729,
      "sft_loss": 1.2916858196258545,
      "step": 300
    },
    {
      "epoch": 0.5249788314987299,
      "grad_norm": 2.628451108932495,
      "learning_rate": 4.631211950800925e-06,
      "logits/chosen": 248.5096435546875,
      "logits/rejected": 248.2730712890625,
      "logps/chosen": -1.3065412044525146,
      "logps/rejected": -1.4278388023376465,
      "loss": 1.3779,
      "odds_ratio_loss": 0.7139269709587097,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13065412640571594,
      "rewards/margins": 0.012129749171435833,
      "rewards/rejected": -0.14278386533260345,
      "sft_loss": 1.3065412044525146,
      "step": 310
    },
    {
      "epoch": 0.541913632514818,
      "grad_norm": 2.022738218307495,
      "learning_rate": 4.6076801701632095e-06,
      "logits/chosen": 250.45730590820312,
      "logits/rejected": 249.79464721679688,
      "logps/chosen": -1.3189256191253662,
      "logps/rejected": -1.4538644552230835,
      "loss": 1.3894,
      "odds_ratio_loss": 0.7043124437332153,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.13189253211021423,
      "rewards/margins": 0.013493897393345833,
      "rewards/rejected": -0.1453864425420761,
      "sft_loss": 1.3189256191253662,
      "step": 320
    },
    {
      "epoch": 0.558848433530906,
      "grad_norm": 1.9951550960540771,
      "learning_rate": 4.583484347456972e-06,
      "logits/chosen": 247.82229614257812,
      "logits/rejected": 248.18814086914062,
      "logps/chosen": -1.4347447156906128,
      "logps/rejected": -1.533195972442627,
      "loss": 1.5114,
      "odds_ratio_loss": 0.7667852640151978,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.14347447454929352,
      "rewards/margins": 0.00984511710703373,
      "rewards/rejected": -0.1533195823431015,
      "sft_loss": 1.4347447156906128,
      "step": 330
    },
    {
      "epoch": 0.5757832345469941,
      "grad_norm": 1.1787029504776,
      "learning_rate": 4.55863210577626e-06,
      "logits/chosen": 248.49844360351562,
      "logits/rejected": 249.04006958007812,
      "logps/chosen": -1.3999149799346924,
      "logps/rejected": -1.6324024200439453,
      "loss": 1.4655,
      "odds_ratio_loss": 0.6553832292556763,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13999152183532715,
      "rewards/margins": 0.02324872463941574,
      "rewards/rejected": -0.1632402390241623,
      "sft_loss": 1.3999149799346924,
      "step": 340
    },
    {
      "epoch": 0.5927180355630821,
      "grad_norm": 1.3341506719589233,
      "learning_rate": 4.5331312750253465e-06,
      "logits/chosen": 250.0833740234375,
      "logits/rejected": 249.986328125,
      "logps/chosen": -1.353328824043274,
      "logps/rejected": -1.4628279209136963,
      "loss": 1.4282,
      "odds_ratio_loss": 0.7489296197891235,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.1353328824043274,
      "rewards/margins": 0.010949901305139065,
      "rewards/rejected": -0.14628279209136963,
      "sft_loss": 1.353328824043274,
      "step": 350
    },
    {
      "epoch": 0.6096528365791702,
      "grad_norm": 7.017007827758789,
      "learning_rate": 4.506989889451858e-06,
      "logits/chosen": 250.78952026367188,
      "logits/rejected": 251.0829315185547,
      "logps/chosen": -1.3330605030059814,
      "logps/rejected": -1.492555022239685,
      "loss": 1.4015,
      "odds_ratio_loss": 0.6845325231552124,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.13330607116222382,
      "rewards/margins": 0.01594943180680275,
      "rewards/rejected": -0.14925549924373627,
      "sft_loss": 1.3330605030059814,
      "step": 360
    },
    {
      "epoch": 0.6265876375952583,
      "grad_norm": 2.496628761291504,
      "learning_rate": 4.480216185115512e-06,
      "logits/chosen": 251.03042602539062,
      "logits/rejected": 251.23403930664062,
      "logps/chosen": -1.3291356563568115,
      "logps/rejected": -1.5497068166732788,
      "loss": 1.3933,
      "odds_ratio_loss": 0.641234278678894,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13291357457637787,
      "rewards/margins": 0.022057104855775833,
      "rewards/rejected": -0.1549706757068634,
      "sft_loss": 1.3291356563568115,
      "step": 370
    },
    {
      "epoch": 0.6435224386113463,
      "grad_norm": 1.7296651601791382,
      "learning_rate": 4.4528185972932856e-06,
      "logits/chosen": 249.67446899414062,
      "logits/rejected": 250.25466918945312,
      "logps/chosen": -1.3678677082061768,
      "logps/rejected": -1.639181137084961,
      "loss": 1.4346,
      "odds_ratio_loss": 0.6676440834999084,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13678675889968872,
      "rewards/margins": 0.02713136002421379,
      "rewards/rejected": -0.1639181226491928,
      "sft_loss": 1.3678677082061768,
      "step": 380
    },
    {
      "epoch": 0.6604572396274344,
      "grad_norm": 5.801638126373291,
      "learning_rate": 4.424805757821803e-06,
      "logits/chosen": 249.99923706054688,
      "logits/rejected": 250.46279907226562,
      "logps/chosen": -1.4075113534927368,
      "logps/rejected": -1.539193868637085,
      "loss": 1.4777,
      "odds_ratio_loss": 0.7016426920890808,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14075112342834473,
      "rewards/margins": 0.013168272562325,
      "rewards/rejected": -0.15391941368579865,
      "sft_loss": 1.4075113534927368,
      "step": 390
    },
    {
      "epoch": 0.6773920406435224,
      "grad_norm": 2.283698558807373,
      "learning_rate": 4.396186492377812e-06,
      "logits/chosen": 250.4367218017578,
      "logits/rejected": 250.3822479248047,
      "logps/chosen": -1.3555715084075928,
      "logps/rejected": -1.6237144470214844,
      "loss": 1.4187,
      "odds_ratio_loss": 0.631491482257843,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1355571299791336,
      "rewards/margins": 0.026814306154847145,
      "rewards/rejected": -0.1623714417219162,
      "sft_loss": 1.3555715084075928,
      "step": 400
    },
    {
      "epoch": 0.6943268416596104,
      "grad_norm": 3.2836830615997314,
      "learning_rate": 4.366969817697578e-06,
      "logits/chosen": 249.8821563720703,
      "logits/rejected": 250.1019744873047,
      "logps/chosen": -1.3262561559677124,
      "logps/rejected": -1.481173038482666,
      "loss": 1.397,
      "odds_ratio_loss": 0.7070230841636658,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.13262560963630676,
      "rewards/margins": 0.015491697005927563,
      "rewards/rejected": -0.1481173038482666,
      "sft_loss": 1.3262561559677124,
      "step": 410
    },
    {
      "epoch": 0.7112616426756986,
      "grad_norm": 2.213815689086914,
      "learning_rate": 4.337164938736086e-06,
      "logits/chosen": 250.5995635986328,
      "logits/rejected": 250.66165161132812,
      "logps/chosen": -1.3863251209259033,
      "logps/rejected": -1.513422966003418,
      "loss": 1.4589,
      "odds_ratio_loss": 0.7256019115447998,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.13863249123096466,
      "rewards/margins": 0.012709791772067547,
      "rewards/rejected": -0.15134228765964508,
      "sft_loss": 1.3863251209259033,
      "step": 420
    },
    {
      "epoch": 0.7281964436917866,
      "grad_norm": 1.0603761672973633,
      "learning_rate": 4.306781245766945e-06,
      "logits/chosen": 250.349853515625,
      "logits/rejected": 250.0968780517578,
      "logps/chosen": -1.2664977312088013,
      "logps/rejected": -1.475754976272583,
      "loss": 1.3367,
      "odds_ratio_loss": 0.7022702097892761,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12664976716041565,
      "rewards/margins": 0.0209257360547781,
      "rewards/rejected": -0.1475754976272583,
      "sft_loss": 1.2664977312088013,
      "step": 430
    },
    {
      "epoch": 0.7451312447078747,
      "grad_norm": 4.412312984466553,
      "learning_rate": 4.275828311423903e-06,
      "logits/chosen": 249.72463989257812,
      "logits/rejected": 250.4665985107422,
      "logps/chosen": -1.4287374019622803,
      "logps/rejected": -1.5396887063980103,
      "loss": 1.4987,
      "odds_ratio_loss": 0.6998986005783081,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.14287373423576355,
      "rewards/margins": 0.011095147579908371,
      "rewards/rejected": -0.15396887063980103,
      "sft_loss": 1.4287374019622803,
      "step": 440
    },
    {
      "epoch": 0.7620660457239627,
      "grad_norm": 6.307819843292236,
      "learning_rate": 4.244315887684912e-06,
      "logits/chosen": 250.8037109375,
      "logits/rejected": 251.0100555419922,
      "logps/chosen": -1.242921233177185,
      "logps/rejected": -1.4474495649337769,
      "loss": 1.3115,
      "odds_ratio_loss": 0.6855745911598206,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12429212033748627,
      "rewards/margins": 0.020452830940485,
      "rewards/rejected": -0.14474496245384216,
      "sft_loss": 1.242921233177185,
      "step": 450
    },
    {
      "epoch": 0.7790008467400508,
      "grad_norm": 1.9661298990249634,
      "learning_rate": 4.212253902799685e-06,
      "logits/chosen": 250.1886749267578,
      "logits/rejected": 250.97207641601562,
      "logps/chosen": -1.3525545597076416,
      "logps/rejected": -1.6002397537231445,
      "loss": 1.4203,
      "odds_ratio_loss": 0.6769625544548035,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13525545597076416,
      "rewards/margins": 0.024768516421318054,
      "rewards/rejected": -0.16002397239208221,
      "sft_loss": 1.3525545597076416,
      "step": 460
    },
    {
      "epoch": 0.7959356477561389,
      "grad_norm": 2.6013875007629395,
      "learning_rate": 4.179652458161718e-06,
      "logits/chosen": 250.9954376220703,
      "logits/rejected": 250.8687286376953,
      "logps/chosen": -1.3839843273162842,
      "logps/rejected": -1.4946000576019287,
      "loss": 1.4574,
      "odds_ratio_loss": 0.7343354821205139,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1383984386920929,
      "rewards/margins": 0.011061567813158035,
      "rewards/rejected": -0.14946000277996063,
      "sft_loss": 1.3839843273162842,
      "step": 470
    },
    {
      "epoch": 0.8128704487722269,
      "grad_norm": 1.4564313888549805,
      "learning_rate": 4.146521825125765e-06,
      "logits/chosen": 251.6759796142578,
      "logits/rejected": 251.0865478515625,
      "logps/chosen": -1.3470927476882935,
      "logps/rejected": -1.5811121463775635,
      "loss": 1.4152,
      "odds_ratio_loss": 0.6814435720443726,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13470926880836487,
      "rewards/margins": 0.023401936516165733,
      "rewards/rejected": -0.15811121463775635,
      "sft_loss": 1.3470927476882935,
      "step": 480
    },
    {
      "epoch": 0.8298052497883149,
      "grad_norm": 2.20600962638855,
      "learning_rate": 4.11287244177176e-06,
      "logits/chosen": 252.32333374023438,
      "logits/rejected": 252.3999481201172,
      "logps/chosen": -1.313819408416748,
      "logps/rejected": -1.5711250305175781,
      "loss": 1.3752,
      "odds_ratio_loss": 0.6139523386955261,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.13138195872306824,
      "rewards/margins": 0.0257305596023798,
      "rewards/rejected": -0.1571125090122223,
      "sft_loss": 1.313819408416748,
      "step": 490
    },
    {
      "epoch": 0.8467400508044031,
      "grad_norm": 6.151379108428955,
      "learning_rate": 4.078714909616215e-06,
      "logits/chosen": 250.7323760986328,
      "logits/rejected": 250.898681640625,
      "logps/chosen": -1.3562355041503906,
      "logps/rejected": -1.6217498779296875,
      "loss": 1.422,
      "odds_ratio_loss": 0.6572010517120361,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.13562354445457458,
      "rewards/margins": 0.026551440358161926,
      "rewards/rejected": -0.1621749848127365,
      "sft_loss": 1.3562355041503906,
      "step": 500
    },
    {
      "epoch": 0.8467400508044031,
      "eval_logits/chosen": 250.27386474609375,
      "eval_logits/rejected": 250.56336975097656,
      "eval_logps/chosen": -1.3222252130508423,
      "eval_logps/rejected": -1.5458606481552124,
      "eval_loss": 1.3895596265792847,
      "eval_odds_ratio_loss": 0.6733438968658447,
      "eval_rewards/accuracies": 0.5752381086349487,
      "eval_rewards/chosen": -0.1322225034236908,
      "eval_rewards/margins": 0.022363554686307907,
      "eval_rewards/rejected": -0.1545860767364502,
      "eval_runtime": 222.405,
      "eval_samples_per_second": 4.721,
      "eval_sft_loss": 1.3222252130508423,
      "eval_steps_per_second": 2.361,
      "step": 500
    },
    {
      "epoch": 0.8636748518204911,
      "grad_norm": 7.651697635650635,
      "learning_rate": 4.044059990272125e-06,
      "logits/chosen": 250.82901000976562,
      "logits/rejected": 250.9202423095703,
      "logps/chosen": -1.3385546207427979,
      "logps/rejected": -1.5796321630477905,
      "loss": 1.4082,
      "odds_ratio_loss": 0.6965717077255249,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.13385547697544098,
      "rewards/margins": 0.024107756093144417,
      "rewards/rejected": -0.15796321630477905,
      "sft_loss": 1.3385546207427979,
      "step": 510
    },
    {
      "epoch": 0.8806096528365792,
      "grad_norm": 2.0727665424346924,
      "learning_rate": 4.0089186020584345e-06,
      "logits/chosen": 250.8845977783203,
      "logits/rejected": 251.09323120117188,
      "logps/chosen": -1.3856604099273682,
      "logps/rejected": -1.521828055381775,
      "loss": 1.453,
      "odds_ratio_loss": 0.6737630367279053,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1385660469532013,
      "rewards/margins": 0.013616764917969704,
      "rewards/rejected": -0.15218280255794525,
      "sft_loss": 1.3856604099273682,
      "step": 520
    },
    {
      "epoch": 0.8975444538526672,
      "grad_norm": 1.6497074365615845,
      "learning_rate": 3.973301816560124e-06,
      "logits/chosen": 250.5475616455078,
      "logits/rejected": 251.07559204101562,
      "logps/chosen": -1.2916876077651978,
      "logps/rejected": -1.5535304546356201,
      "loss": 1.355,
      "odds_ratio_loss": 0.6329156160354614,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.12916876375675201,
      "rewards/margins": 0.026184294372797012,
      "rewards/rejected": -0.15535303950309753,
      "sft_loss": 1.2916876077651978,
      "step": 530
    },
    {
      "epoch": 0.9144792548687553,
      "grad_norm": 2.159715414047241,
      "learning_rate": 3.937220855140021e-06,
      "logits/chosen": 252.92514038085938,
      "logits/rejected": 252.7586212158203,
      "logps/chosen": -1.2703886032104492,
      "logps/rejected": -1.4452732801437378,
      "loss": 1.3383,
      "odds_ratio_loss": 0.6788212060928345,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1270388662815094,
      "rewards/margins": 0.017488475888967514,
      "rewards/rejected": -0.14452733099460602,
      "sft_loss": 1.2703886032104492,
      "step": 540
    },
    {
      "epoch": 0.9314140558848434,
      "grad_norm": 1.6743594408035278,
      "learning_rate": 3.900687085403418e-06,
      "logits/chosen": 251.9027099609375,
      "logits/rejected": 251.82095336914062,
      "logps/chosen": -1.2356189489364624,
      "logps/rejected": -1.3227603435516357,
      "loss": 1.3057,
      "odds_ratio_loss": 0.7003273963928223,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12356189638376236,
      "rewards/margins": 0.008714134804904461,
      "rewards/rejected": -0.1322760283946991,
      "sft_loss": 1.2356189489364624,
      "step": 550
    },
    {
      "epoch": 0.9483488569009314,
      "grad_norm": 2.0650203227996826,
      "learning_rate": 3.863712017616614e-06,
      "logits/chosen": 252.4972686767578,
      "logits/rejected": 253.4659881591797,
      "logps/chosen": -1.33904230594635,
      "logps/rejected": -1.5850245952606201,
      "loss": 1.4043,
      "odds_ratio_loss": 0.6524351835250854,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.13390421867370605,
      "rewards/margins": 0.024598244577646255,
      "rewards/rejected": -0.158502459526062,
      "sft_loss": 1.33904230594635,
      "step": 560
    },
    {
      "epoch": 0.9652836579170194,
      "grad_norm": 4.395691394805908,
      "learning_rate": 3.826307301080504e-06,
      "logits/chosen": 250.24227905273438,
      "logits/rejected": 250.7295379638672,
      "logps/chosen": -1.3605867624282837,
      "logps/rejected": -1.592639684677124,
      "loss": 1.436,
      "odds_ratio_loss": 0.7545104622840881,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13605865836143494,
      "rewards/margins": 0.02320530079305172,
      "rewards/rejected": -0.1592639982700348,
      "sft_loss": 1.3605867624282837,
      "step": 570
    },
    {
      "epoch": 0.9822184589331076,
      "grad_norm": 2.8619537353515625,
      "learning_rate": 3.7884847204603775e-06,
      "logits/chosen": 252.0938262939453,
      "logits/rejected": 252.2036895751953,
      "logps/chosen": -1.2960541248321533,
      "logps/rejected": -1.4377543926239014,
      "loss": 1.3661,
      "odds_ratio_loss": 0.7007730007171631,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.12960541248321533,
      "rewards/margins": 0.014170033857226372,
      "rewards/rejected": -0.14377544820308685,
      "sft_loss": 1.2960541248321533,
      "step": 580
    },
    {
      "epoch": 0.9991532599491956,
      "grad_norm": 2.728938102722168,
      "learning_rate": 3.750256192073058e-06,
      "logits/chosen": 252.297607421875,
      "logits/rejected": 252.3206024169922,
      "logps/chosen": -1.4331130981445312,
      "logps/rejected": -1.4828779697418213,
      "loss": 1.5094,
      "odds_ratio_loss": 0.7630207538604736,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.1433113068342209,
      "rewards/margins": 0.004976483527570963,
      "rewards/rejected": -0.1482878029346466,
      "sft_loss": 1.4331130981445312,
      "step": 590
    },
    {
      "epoch": 1.0160880609652836,
      "grad_norm": 4.769712924957275,
      "learning_rate": 3.7116337601325715e-06,
      "logits/chosen": 251.11611938476562,
      "logits/rejected": 251.3095245361328,
      "logps/chosen": -1.2182286977767944,
      "logps/rejected": -1.3984460830688477,
      "loss": 1.285,
      "odds_ratio_loss": 0.6677871346473694,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12182287871837616,
      "rewards/margins": 0.01802174374461174,
      "rewards/rejected": -0.139844611287117,
      "sft_loss": 1.2182286977767944,
      "step": 600
    },
    {
      "epoch": 1.0330228619813717,
      "grad_norm": 2.1474721431732178,
      "learning_rate": 3.6726295929555154e-06,
      "logits/chosen": 251.42453002929688,
      "logits/rejected": 252.20944213867188,
      "logps/chosen": -1.2515238523483276,
      "logps/rejected": -1.3369293212890625,
      "loss": 1.3216,
      "odds_ratio_loss": 0.7008894085884094,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12515239417552948,
      "rewards/margins": 0.008540543727576733,
      "rewards/rejected": -0.1336929351091385,
      "sft_loss": 1.2515238523483276,
      "step": 610
    },
    {
      "epoch": 1.0499576629974599,
      "grad_norm": 2.5692148208618164,
      "learning_rate": 3.6332559791273307e-06,
      "logits/chosen": 251.50045776367188,
      "logits/rejected": 251.7969970703125,
      "logps/chosen": -1.236567735671997,
      "logps/rejected": -1.4078432321548462,
      "loss": 1.3014,
      "odds_ratio_loss": 0.6482684016227722,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.12365677207708359,
      "rewards/margins": 0.01712755486369133,
      "rewards/rejected": -0.14078432321548462,
      "sft_loss": 1.236567735671997,
      "step": 620
    },
    {
      "epoch": 1.0668924640135478,
      "grad_norm": 3.342146635055542,
      "learning_rate": 3.593525323630681e-06,
      "logits/chosen": 251.15097045898438,
      "logits/rejected": 251.6619110107422,
      "logps/chosen": -1.2767521142959595,
      "logps/rejected": -1.4714807271957397,
      "loss": 1.34,
      "odds_ratio_loss": 0.6320163607597351,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12767520546913147,
      "rewards/margins": 0.019472863525152206,
      "rewards/rejected": -0.14714807271957397,
      "sft_loss": 1.2767521142959595,
      "step": 630
    },
    {
      "epoch": 1.083827265029636,
      "grad_norm": 1.7750215530395508,
      "learning_rate": 3.5534501439371615e-06,
      "logits/chosen": 253.18130493164062,
      "logits/rejected": 253.9321746826172,
      "logps/chosen": -1.2172341346740723,
      "logps/rejected": -1.4529893398284912,
      "loss": 1.282,
      "odds_ratio_loss": 0.6480832099914551,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12172339856624603,
      "rewards/margins": 0.023575523868203163,
      "rewards/rejected": -0.14529892802238464,
      "sft_loss": 1.2172341346740723,
      "step": 640
    },
    {
      "epoch": 1.100762066045724,
      "grad_norm": 3.0737292766571045,
      "learning_rate": 3.5130430660635633e-06,
      "logits/chosen": 253.6792449951172,
      "logits/rejected": 253.87606811523438,
      "logps/chosen": -1.2902581691741943,
      "logps/rejected": -1.4731221199035645,
      "loss": 1.3556,
      "odds_ratio_loss": 0.6539097428321838,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12902581691741943,
      "rewards/margins": 0.01828640140593052,
      "rewards/rejected": -0.1473122239112854,
      "sft_loss": 1.2902581691741943,
      "step": 650
    },
    {
      "epoch": 1.117696867061812,
      "grad_norm": 1.3352795839309692,
      "learning_rate": 3.4723168205939444e-06,
      "logits/chosen": 252.404541015625,
      "logits/rejected": 252.8050994873047,
      "logps/chosen": -1.2593786716461182,
      "logps/rejected": -1.3454132080078125,
      "loss": 1.3296,
      "odds_ratio_loss": 0.7021333575248718,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12593786418437958,
      "rewards/margins": 0.008603455498814583,
      "rewards/rejected": -0.134541317820549,
      "sft_loss": 1.2593786716461182,
      "step": 660
    },
    {
      "epoch": 1.1346316680779,
      "grad_norm": 1.4886754751205444,
      "learning_rate": 3.431284238668754e-06,
      "logits/chosen": 252.7022247314453,
      "logits/rejected": 253.0660858154297,
      "logps/chosen": -1.344042420387268,
      "logps/rejected": -1.5089060068130493,
      "loss": 1.4141,
      "odds_ratio_loss": 0.7006558775901794,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.134404256939888,
      "rewards/margins": 0.01648634299635887,
      "rewards/rejected": -0.15089061856269836,
      "sft_loss": 1.344042420387268,
      "step": 670
    },
    {
      "epoch": 1.1515664690939882,
      "grad_norm": 8.335692405700684,
      "learning_rate": 3.389958247942274e-06,
      "logits/chosen": 251.40444946289062,
      "logits/rejected": 252.54983520507812,
      "logps/chosen": -1.358946442604065,
      "logps/rejected": -1.5931769609451294,
      "loss": 1.4306,
      "odds_ratio_loss": 0.7168671488761902,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13589465618133545,
      "rewards/margins": 0.023423049598932266,
      "rewards/rejected": -0.15931770205497742,
      "sft_loss": 1.358946442604065,
      "step": 680
    },
    {
      "epoch": 1.168501270110076,
      "grad_norm": 2.2777557373046875,
      "learning_rate": 3.3483518685096588e-06,
      "logits/chosen": 252.7519989013672,
      "logits/rejected": 252.7262420654297,
      "logps/chosen": -1.313045859336853,
      "logps/rejected": -1.4913846254348755,
      "loss": 1.3792,
      "odds_ratio_loss": 0.6616145372390747,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13130459189414978,
      "rewards/margins": 0.017833886668086052,
      "rewards/rejected": -0.14913848042488098,
      "sft_loss": 1.313045859336853,
      "step": 690
    },
    {
      "epoch": 1.1854360711261642,
      "grad_norm": 1.7483341693878174,
      "learning_rate": 3.306478208804839e-06,
      "logits/chosen": 251.73300170898438,
      "logits/rejected": 252.00765991210938,
      "logps/chosen": -1.3198022842407227,
      "logps/rejected": -1.5368399620056152,
      "loss": 1.3902,
      "odds_ratio_loss": 0.7039870023727417,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.13198022544384003,
      "rewards/margins": 0.02170378342270851,
      "rewards/rejected": -0.15368400514125824,
      "sft_loss": 1.3198022842407227,
      "step": 700
    },
    {
      "epoch": 1.2023708721422524,
      "grad_norm": 3.1045162677764893,
      "learning_rate": 3.264350461470608e-06,
      "logits/chosen": 251.3062744140625,
      "logits/rejected": 251.24673461914062,
      "logps/chosen": -1.191873550415039,
      "logps/rejected": -1.5103055238723755,
      "loss": 1.2531,
      "odds_ratio_loss": 0.6123217344284058,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1191873699426651,
      "rewards/margins": 0.031843192875385284,
      "rewards/rejected": -0.15103057026863098,
      "sft_loss": 1.191873550415039,
      "step": 710
    },
    {
      "epoch": 1.2193056731583405,
      "grad_norm": 4.187674522399902,
      "learning_rate": 3.2219818992021685e-06,
      "logits/chosen": 250.6509246826172,
      "logits/rejected": 250.8859100341797,
      "logps/chosen": -1.1646994352340698,
      "logps/rejected": -1.4936308860778809,
      "loss": 1.2266,
      "odds_ratio_loss": 0.6186606884002686,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.11646995693445206,
      "rewards/margins": 0.032893143594264984,
      "rewards/rejected": -0.14936310052871704,
      "sft_loss": 1.1646994352340698,
      "step": 720
    },
    {
      "epoch": 1.2362404741744284,
      "grad_norm": 6.236975193023682,
      "learning_rate": 3.1793858705654595e-06,
      "logits/chosen": 252.42489624023438,
      "logits/rejected": 252.6690673828125,
      "logps/chosen": -1.1537976264953613,
      "logps/rejected": -1.407354712486267,
      "loss": 1.2166,
      "odds_ratio_loss": 0.6279497146606445,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.11537975072860718,
      "rewards/margins": 0.02535572089254856,
      "rewards/rejected": -0.1407354772090912,
      "sft_loss": 1.1537976264953613,
      "step": 730
    },
    {
      "epoch": 1.2531752751905165,
      "grad_norm": 4.287051677703857,
      "learning_rate": 3.1365757957915787e-06,
      "logits/chosen": 252.34127807617188,
      "logits/rejected": 252.6292724609375,
      "logps/chosen": -1.2895433902740479,
      "logps/rejected": -1.4524219036102295,
      "loss": 1.3549,
      "odds_ratio_loss": 0.6531358957290649,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12895433604717255,
      "rewards/margins": 0.016287846490740776,
      "rewards/rejected": -0.14524218440055847,
      "sft_loss": 1.2895433902740479,
      "step": 740
    },
    {
      "epoch": 1.2701100762066047,
      "grad_norm": 3.4889471530914307,
      "learning_rate": 3.093565162548633e-06,
      "logits/chosen": 252.25082397460938,
      "logits/rejected": 252.3234405517578,
      "logps/chosen": -1.3181928396224976,
      "logps/rejected": -1.5464909076690674,
      "loss": 1.3886,
      "odds_ratio_loss": 0.7041035890579224,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13181930780410767,
      "rewards/margins": 0.02282978780567646,
      "rewards/rejected": -0.15464909374713898,
      "sft_loss": 1.3181928396224976,
      "step": 750
    },
    {
      "epoch": 1.2870448772226926,
      "grad_norm": 2.6276607513427734,
      "learning_rate": 3.0503675216923294e-06,
      "logits/chosen": 252.7392120361328,
      "logits/rejected": 253.15933227539062,
      "logps/chosen": -1.1722157001495361,
      "logps/rejected": -1.3820818662643433,
      "loss": 1.2364,
      "odds_ratio_loss": 0.642305850982666,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11722154915332794,
      "rewards/margins": 0.020986629649996758,
      "rewards/rejected": -0.13820818066596985,
      "sft_loss": 1.1722157001495361,
      "step": 760
    },
    {
      "epoch": 1.3039796782387807,
      "grad_norm": 2.102112054824829,
      "learning_rate": 3.0069964829966748e-06,
      "logits/chosen": 251.8651885986328,
      "logits/rejected": 252.33065795898438,
      "logps/chosen": -1.238761305809021,
      "logps/rejected": -1.385432481765747,
      "loss": 1.3081,
      "odds_ratio_loss": 0.6930680274963379,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.12387613952159882,
      "rewards/margins": 0.014667103998363018,
      "rewards/rejected": -0.1385432332754135,
      "sft_loss": 1.238761305809021,
      "step": 770
    },
    {
      "epoch": 1.3209144792548688,
      "grad_norm": 1.8301111459732056,
      "learning_rate": 2.963465710866094e-06,
      "logits/chosen": 251.42507934570312,
      "logits/rejected": 251.70413208007812,
      "logps/chosen": -1.2377374172210693,
      "logps/rejected": -1.572333574295044,
      "loss": 1.2996,
      "odds_ratio_loss": 0.6182882189750671,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.1237737163901329,
      "rewards/margins": 0.0334596149623394,
      "rewards/rejected": -0.1572333723306656,
      "sft_loss": 1.2377374172210693,
      "step": 780
    },
    {
      "epoch": 1.337849280270957,
      "grad_norm": 7.058846473693848,
      "learning_rate": 2.919788920030357e-06,
      "logits/chosen": 253.6296844482422,
      "logits/rejected": 254.1488800048828,
      "logps/chosen": -1.2885361909866333,
      "logps/rejected": -1.4078810214996338,
      "loss": 1.3598,
      "odds_ratio_loss": 0.7127668857574463,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.12885363399982452,
      "rewards/margins": 0.011934494599699974,
      "rewards/rejected": -0.14078812301158905,
      "sft_loss": 1.2885361909866333,
      "step": 790
    },
    {
      "epoch": 1.3547840812870449,
      "grad_norm": 3.2417685985565186,
      "learning_rate": 2.8759798712236303e-06,
      "logits/chosen": 253.9468231201172,
      "logits/rejected": 254.49008178710938,
      "logps/chosen": -1.2510448694229126,
      "logps/rejected": -1.554084300994873,
      "loss": 1.3164,
      "odds_ratio_loss": 0.6536312103271484,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12510448694229126,
      "rewards/margins": 0.030303955078125,
      "rewards/rejected": -0.15540844202041626,
      "sft_loss": 1.2510448694229126,
      "step": 800
    },
    {
      "epoch": 1.371718882303133,
      "grad_norm": 1.9042048454284668,
      "learning_rate": 2.8320523668490507e-06,
      "logits/chosen": 253.7815704345703,
      "logits/rejected": 254.0332489013672,
      "logps/chosen": -1.2883363962173462,
      "logps/rejected": -1.4249004125595093,
      "loss": 1.3613,
      "odds_ratio_loss": 0.7295271754264832,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12883363664150238,
      "rewards/margins": 0.01365639828145504,
      "rewards/rejected": -0.14249004423618317,
      "sft_loss": 1.2883363962173462,
      "step": 810
    },
    {
      "epoch": 1.388653683319221,
      "grad_norm": 3.2637712955474854,
      "learning_rate": 2.7880202466301597e-06,
      "logits/chosen": 253.0078125,
      "logits/rejected": 253.09902954101562,
      "logps/chosen": -1.2434396743774414,
      "logps/rejected": -1.434788465499878,
      "loss": 1.3141,
      "odds_ratio_loss": 0.7062419652938843,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12434396892786026,
      "rewards/margins": 0.019134875386953354,
      "rewards/rejected": -0.14347884058952332,
      "sft_loss": 1.2434396743774414,
      "step": 820
    },
    {
      "epoch": 1.405588484335309,
      "grad_norm": 2.8521387577056885,
      "learning_rate": 2.7438973832505854e-06,
      "logits/chosen": 251.6912841796875,
      "logits/rejected": 252.3306121826172,
      "logps/chosen": -1.175612211227417,
      "logps/rejected": -1.4439500570297241,
      "loss": 1.2387,
      "odds_ratio_loss": 0.6307461857795715,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.1175612211227417,
      "rewards/margins": 0.026833802461624146,
      "rewards/rejected": -0.14439500868320465,
      "sft_loss": 1.175612211227417,
      "step": 830
    },
    {
      "epoch": 1.4225232853513972,
      "grad_norm": 8.145666122436523,
      "learning_rate": 2.699697677983341e-06,
      "logits/chosen": 253.7397918701172,
      "logits/rejected": 253.99203491210938,
      "logps/chosen": -1.2202482223510742,
      "logps/rejected": -1.2863436937332153,
      "loss": 1.2952,
      "odds_ratio_loss": 0.7496277093887329,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.12202481925487518,
      "rewards/margins": 0.006609548814594746,
      "rewards/rejected": -0.12863437831401825,
      "sft_loss": 1.2202482223510742,
      "step": 840
    },
    {
      "epoch": 1.4394580863674853,
      "grad_norm": 8.698111534118652,
      "learning_rate": 2.6554350563111115e-06,
      "logits/chosen": 252.4715118408203,
      "logits/rejected": 252.827392578125,
      "logps/chosen": -1.286290168762207,
      "logps/rejected": -1.4043002128601074,
      "loss": 1.3605,
      "odds_ratio_loss": 0.7424478530883789,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12862902879714966,
      "rewards/margins": 0.011800997890532017,
      "rewards/rejected": -0.1404300183057785,
      "sft_loss": 1.286290168762207,
      "step": 850
    },
    {
      "epoch": 1.4563928873835732,
      "grad_norm": 5.0890398025512695,
      "learning_rate": 2.611123463538913e-06,
      "logits/chosen": 251.801513671875,
      "logits/rejected": 251.6995391845703,
      "logps/chosen": -1.1361697912216187,
      "logps/rejected": -1.4048140048980713,
      "loss": 1.1998,
      "odds_ratio_loss": 0.6365838050842285,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.11361698061227798,
      "rewards/margins": 0.02686440572142601,
      "rewards/rejected": -0.1404813826084137,
      "sft_loss": 1.1361697912216187,
      "step": 860
    },
    {
      "epoch": 1.4733276883996613,
      "grad_norm": 2.2515621185302734,
      "learning_rate": 2.566776860400514e-06,
      "logits/chosen": 253.1812286376953,
      "logits/rejected": 253.3629913330078,
      "logps/chosen": -1.3416874408721924,
      "logps/rejected": -1.5248029232025146,
      "loss": 1.4085,
      "odds_ratio_loss": 0.6685901880264282,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13416874408721924,
      "rewards/margins": 0.01831156201660633,
      "rewards/rejected": -0.15248030424118042,
      "sft_loss": 1.3416874408721924,
      "step": 870
    },
    {
      "epoch": 1.4902624894157492,
      "grad_norm": 1.626927137374878,
      "learning_rate": 2.522409218659989e-06,
      "logits/chosen": 254.90966796875,
      "logits/rejected": 255.2332305908203,
      "logps/chosen": -1.2846667766571045,
      "logps/rejected": -1.4117066860198975,
      "loss": 1.3537,
      "odds_ratio_loss": 0.6902014017105103,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.1284666806459427,
      "rewards/margins": 0.012703998014330864,
      "rewards/rejected": -0.1411706656217575,
      "sft_loss": 1.2846667766571045,
      "step": 880
    },
    {
      "epoch": 1.5071972904318374,
      "grad_norm": 2.0912625789642334,
      "learning_rate": 2.4780345167097976e-06,
      "logits/chosen": 252.5232696533203,
      "logits/rejected": 253.27197265625,
      "logps/chosen": -1.2608062028884888,
      "logps/rejected": -1.5506179332733154,
      "loss": 1.3254,
      "odds_ratio_loss": 0.6455532908439636,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12608060240745544,
      "rewards/margins": 0.02898118831217289,
      "rewards/rejected": -0.15506179630756378,
      "sft_loss": 1.2608062028884888,
      "step": 890
    },
    {
      "epoch": 1.5241320914479255,
      "grad_norm": 2.230842351913452,
      "learning_rate": 2.4336667351667747e-06,
      "logits/chosen": 254.10025024414062,
      "logits/rejected": 254.30361938476562,
      "logps/chosen": -1.2958338260650635,
      "logps/rejected": -1.5468194484710693,
      "loss": 1.358,
      "odds_ratio_loss": 0.6219068765640259,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.12958340346813202,
      "rewards/margins": 0.025098532438278198,
      "rewards/rejected": -0.1546819508075714,
      "sft_loss": 1.2958338260650635,
      "step": 900
    },
    {
      "epoch": 1.5410668924640136,
      "grad_norm": 4.812232494354248,
      "learning_rate": 2.3893198524674264e-06,
      "logits/chosen": 252.809814453125,
      "logits/rejected": 252.73892211914062,
      "logps/chosen": -1.2310715913772583,
      "logps/rejected": -1.428946852684021,
      "loss": 1.2975,
      "odds_ratio_loss": 0.6640914678573608,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12310715764760971,
      "rewards/margins": 0.019787531346082687,
      "rewards/rejected": -0.1428947001695633,
      "sft_loss": 1.2310715913772583,
      "step": 910
    },
    {
      "epoch": 1.5580016934801018,
      "grad_norm": 2.730430841445923,
      "learning_rate": 2.345007840463904e-06,
      "logits/chosen": 254.012939453125,
      "logits/rejected": 254.33950805664062,
      "logps/chosen": -1.2894508838653564,
      "logps/rejected": -1.4280718564987183,
      "loss": 1.3585,
      "odds_ratio_loss": 0.6902931332588196,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12894509732723236,
      "rewards/margins": 0.013862092979252338,
      "rewards/rejected": -0.14280718564987183,
      "sft_loss": 1.2894508838653564,
      "step": 920
    },
    {
      "epoch": 1.5749364944961897,
      "grad_norm": 3.1161863803863525,
      "learning_rate": 2.3007446600220572e-06,
      "logits/chosen": 254.1707305908203,
      "logits/rejected": 253.95748901367188,
      "logps/chosen": -1.2720592021942139,
      "logps/rejected": -1.4615156650543213,
      "loss": 1.3436,
      "odds_ratio_loss": 0.7155615091323853,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.12720590829849243,
      "rewards/margins": 0.018945645540952682,
      "rewards/rejected": -0.1461515724658966,
      "sft_loss": 1.2720592021942139,
      "step": 930
    },
    {
      "epoch": 1.5918712955122776,
      "grad_norm": 2.035860538482666,
      "learning_rate": 2.2565442566229507e-06,
      "logits/chosen": 253.0424346923828,
      "logits/rejected": 253.29043579101562,
      "logps/chosen": -1.2878587245941162,
      "logps/rejected": -1.4205825328826904,
      "loss": 1.3592,
      "odds_ratio_loss": 0.7135877013206482,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.1287858784198761,
      "rewards/margins": 0.013272365555167198,
      "rewards/rejected": -0.14205823838710785,
      "sft_loss": 1.2878587245941162,
      "step": 940
    },
    {
      "epoch": 1.6088060965283657,
      "grad_norm": 2.338395357131958,
      "learning_rate": 2.2124205559692195e-06,
      "logits/chosen": 253.37295532226562,
      "logits/rejected": 253.95040893554688,
      "logps/chosen": -1.2591735124588013,
      "logps/rejected": -1.4865353107452393,
      "loss": 1.3211,
      "odds_ratio_loss": 0.6195861101150513,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.12591736018657684,
      "rewards/margins": 0.02273617871105671,
      "rewards/rejected": -0.1486535370349884,
      "sft_loss": 1.2591735124588013,
      "step": 950
    },
    {
      "epoch": 1.6257408975444538,
      "grad_norm": 3.2883460521698,
      "learning_rate": 2.168387459597666e-06,
      "logits/chosen": 253.46572875976562,
      "logits/rejected": 253.340576171875,
      "logps/chosen": -1.2676080465316772,
      "logps/rejected": -1.5205793380737305,
      "loss": 1.3311,
      "odds_ratio_loss": 0.6347194910049438,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.1267608106136322,
      "rewards/margins": 0.025297129526734352,
      "rewards/rejected": -0.1520579308271408,
      "sft_loss": 1.2676080465316772,
      "step": 960
    },
    {
      "epoch": 1.642675698560542,
      "grad_norm": 3.6740529537200928,
      "learning_rate": 2.1244588404994648e-06,
      "logits/chosen": 251.4237518310547,
      "logits/rejected": 251.6488494873047,
      "logps/chosen": -1.2188889980316162,
      "logps/rejected": -1.379131555557251,
      "loss": 1.2882,
      "odds_ratio_loss": 0.693446695804596,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1218889132142067,
      "rewards/margins": 0.016024235635995865,
      "rewards/rejected": -0.13791313767433167,
      "sft_loss": 1.2188889980316162,
      "step": 970
    },
    {
      "epoch": 1.65961049957663,
      "grad_norm": 6.992788791656494,
      "learning_rate": 2.08064853874936e-06,
      "logits/chosen": 253.6184539794922,
      "logits/rejected": 253.8686981201172,
      "logps/chosen": -1.2761471271514893,
      "logps/rejected": -1.4780161380767822,
      "loss": 1.3423,
      "odds_ratio_loss": 0.6611601114273071,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12761470675468445,
      "rewards/margins": 0.020186906680464745,
      "rewards/rejected": -0.14780160784721375,
      "sft_loss": 1.2761471271514893,
      "step": 980
    },
    {
      "epoch": 1.676545300592718,
      "grad_norm": 4.474498748779297,
      "learning_rate": 2.0369703571452387e-06,
      "logits/chosen": 253.08810424804688,
      "logits/rejected": 253.7334747314453,
      "logps/chosen": -1.1575729846954346,
      "logps/rejected": -1.4460541009902954,
      "loss": 1.2163,
      "odds_ratio_loss": 0.5876318216323853,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.1157573014497757,
      "rewards/margins": 0.02884811721742153,
      "rewards/rejected": -0.14460542798042297,
      "sft_loss": 1.1575729846954346,
      "step": 990
    },
    {
      "epoch": 1.6934801016088061,
      "grad_norm": 2.3749420642852783,
      "learning_rate": 1.993438056859441e-06,
      "logits/chosen": 254.9521026611328,
      "logits/rejected": 255.0730743408203,
      "logps/chosen": -1.2478586435317993,
      "logps/rejected": -1.4748018980026245,
      "loss": 1.3103,
      "odds_ratio_loss": 0.6246139407157898,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.124785877764225,
      "rewards/margins": 0.022694315761327744,
      "rewards/rejected": -0.14748017489910126,
      "sft_loss": 1.2478586435317993,
      "step": 1000
    },
    {
      "epoch": 1.6934801016088061,
      "eval_logits/chosen": 252.814697265625,
      "eval_logits/rejected": 253.1349639892578,
      "eval_logps/chosen": -1.2641632556915283,
      "eval_logps/rejected": -1.4885586500167847,
      "eval_loss": 1.3313392400741577,
      "eval_odds_ratio_loss": 0.6717599630355835,
      "eval_rewards/accuracies": 0.569523811340332,
      "eval_rewards/chosen": -0.12641634047031403,
      "eval_rewards/margins": 0.022439539432525635,
      "eval_rewards/rejected": -0.14885587990283966,
      "eval_runtime": 221.9589,
      "eval_samples_per_second": 4.731,
      "eval_sft_loss": 1.2641632556915283,
      "eval_steps_per_second": 2.365,
      "step": 1000
    },
    {
      "epoch": 1.710414902624894,
      "grad_norm": 2.523041009902954,
      "learning_rate": 1.9500653531031917e-06,
      "logits/chosen": 253.83682250976562,
      "logits/rejected": 254.0968780517578,
      "logps/chosen": -1.231729507446289,
      "logps/rejected": -1.4748117923736572,
      "loss": 1.2981,
      "odds_ratio_loss": 0.6637791395187378,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12317295372486115,
      "rewards/margins": 0.02430824190378189,
      "rewards/rejected": -0.14748118817806244,
      "sft_loss": 1.231729507446289,
      "step": 1010
    },
    {
      "epoch": 1.7273497036409822,
      "grad_norm": 2.989529848098755,
      "learning_rate": 1.9068659108055117e-06,
      "logits/chosen": 252.50570678710938,
      "logits/rejected": 252.9430389404297,
      "logps/chosen": -1.2413790225982666,
      "logps/rejected": -1.4306474924087524,
      "loss": 1.3076,
      "odds_ratio_loss": 0.6622155904769897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12413790076971054,
      "rewards/margins": 0.018926845863461494,
      "rewards/rejected": -0.14306475222110748,
      "sft_loss": 1.2413790225982666,
      "step": 1020
    },
    {
      "epoch": 1.7442845046570703,
      "grad_norm": 1.9568462371826172,
      "learning_rate": 1.863853340307962e-06,
      "logits/chosen": 252.41305541992188,
      "logits/rejected": 253.2246551513672,
      "logps/chosen": -1.0862369537353516,
      "logps/rejected": -1.4562585353851318,
      "loss": 1.1465,
      "odds_ratio_loss": 0.6029458045959473,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.1086236983537674,
      "rewards/margins": 0.037002164870500565,
      "rewards/rejected": -0.14562585949897766,
      "sft_loss": 1.0862369537353516,
      "step": 1030
    },
    {
      "epoch": 1.7612193056731584,
      "grad_norm": 2.321819543838501,
      "learning_rate": 1.8210411930766019e-06,
      "logits/chosen": 253.8018035888672,
      "logits/rejected": 253.84048461914062,
      "logps/chosen": -1.2366716861724854,
      "logps/rejected": -1.473456621170044,
      "loss": 1.3013,
      "odds_ratio_loss": 0.6460654139518738,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.1236671656370163,
      "rewards/margins": 0.023678502067923546,
      "rewards/rejected": -0.1473456621170044,
      "sft_loss": 1.2366716861724854,
      "step": 1040
    },
    {
      "epoch": 1.7781541066892466,
      "grad_norm": 2.2399725914001465,
      "learning_rate": 1.7784429574324803e-06,
      "logits/chosen": 253.9780731201172,
      "logits/rejected": 253.922607421875,
      "logps/chosen": -1.199683427810669,
      "logps/rejected": -1.4928243160247803,
      "loss": 1.2597,
      "odds_ratio_loss": 0.599722683429718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11996833980083466,
      "rewards/margins": 0.02931409515440464,
      "rewards/rejected": -0.14928242564201355,
      "sft_loss": 1.199683427810669,
      "step": 1050
    },
    {
      "epoch": 1.7950889077053345,
      "grad_norm": 1.5600874423980713,
      "learning_rate": 1.7360720543020327e-06,
      "logits/chosen": 254.2810516357422,
      "logits/rejected": 254.4838409423828,
      "logps/chosen": -1.2215511798858643,
      "logps/rejected": -1.3866732120513916,
      "loss": 1.2883,
      "odds_ratio_loss": 0.6674426794052124,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12215511500835419,
      "rewards/margins": 0.016512203961610794,
      "rewards/rejected": -0.13866731524467468,
      "sft_loss": 1.2215511798858643,
      "step": 1060
    },
    {
      "epoch": 1.8120237087214224,
      "grad_norm": 6.640655994415283,
      "learning_rate": 1.6939418329887042e-06,
      "logits/chosen": 254.4890594482422,
      "logits/rejected": 254.67626953125,
      "logps/chosen": -1.2743191719055176,
      "logps/rejected": -1.505152940750122,
      "loss": 1.3393,
      "odds_ratio_loss": 0.6494858264923096,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12743191421031952,
      "rewards/margins": 0.023083385080099106,
      "rewards/rejected": -0.15051528811454773,
      "sft_loss": 1.2743191719055176,
      "step": 1070
    },
    {
      "epoch": 1.8289585097375105,
      "grad_norm": 2.518933057785034,
      "learning_rate": 1.6520655669671467e-06,
      "logits/chosen": 254.40719604492188,
      "logits/rejected": 254.98049926757812,
      "logps/chosen": -1.2717182636260986,
      "logps/rejected": -1.383216142654419,
      "loss": 1.3404,
      "odds_ratio_loss": 0.686890721321106,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.1271718293428421,
      "rewards/margins": 0.011149783618748188,
      "rewards/rejected": -0.13832160830497742,
      "sft_loss": 1.2717182636260986,
      "step": 1080
    },
    {
      "epoch": 1.8458933107535986,
      "grad_norm": 2.115421772003174,
      "learning_rate": 1.610456449701294e-06,
      "logits/chosen": 254.34115600585938,
      "logits/rejected": 254.33877563476562,
      "logps/chosen": -1.2869365215301514,
      "logps/rejected": -1.4801461696624756,
      "loss": 1.356,
      "odds_ratio_loss": 0.6902877688407898,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12869365513324738,
      "rewards/margins": 0.019320974126458168,
      "rewards/rejected": -0.1480146199464798,
      "sft_loss": 1.2869365215301514,
      "step": 1090
    },
    {
      "epoch": 1.8628281117696868,
      "grad_norm": 3.411358118057251,
      "learning_rate": 1.5691275904876545e-06,
      "logits/chosen": 253.40380859375,
      "logits/rejected": 253.9809112548828,
      "logps/chosen": -1.3146874904632568,
      "logps/rejected": -1.605054497718811,
      "loss": 1.3787,
      "odds_ratio_loss": 0.6398962140083313,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.1314687579870224,
      "rewards/margins": 0.02903667651116848,
      "rewards/rejected": -0.16050544381141663,
      "sft_loss": 1.3146874904632568,
      "step": 1100
    },
    {
      "epoch": 1.879762912785775,
      "grad_norm": 2.1282906532287598,
      "learning_rate": 1.5280920103251235e-06,
      "logits/chosen": 254.62075805664062,
      "logits/rejected": 255.1248321533203,
      "logps/chosen": -1.219244360923767,
      "logps/rejected": -1.4468841552734375,
      "loss": 1.2855,
      "odds_ratio_loss": 0.6621590852737427,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12192445993423462,
      "rewards/margins": 0.022763973101973534,
      "rewards/rejected": -0.1446884274482727,
      "sft_loss": 1.219244360923767,
      "step": 1110
    },
    {
      "epoch": 1.8966977138018628,
      "grad_norm": 3.646892786026001,
      "learning_rate": 1.4873626378126015e-06,
      "logits/chosen": 253.64559936523438,
      "logits/rejected": 253.7223663330078,
      "logps/chosen": -1.2419589757919312,
      "logps/rejected": -1.5452362298965454,
      "loss": 1.3108,
      "odds_ratio_loss": 0.6882845759391785,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12419591099023819,
      "rewards/margins": 0.03032771870493889,
      "rewards/rejected": -0.15452361106872559,
      "sft_loss": 1.2419589757919312,
      "step": 1120
    },
    {
      "epoch": 1.913632514817951,
      "grad_norm": 2.910693407058716,
      "learning_rate": 1.446952305075738e-06,
      "logits/chosen": 255.579833984375,
      "logits/rejected": 255.78054809570312,
      "logps/chosen": -1.220640778541565,
      "logps/rejected": -1.3528368473052979,
      "loss": 1.2889,
      "odds_ratio_loss": 0.6829566359519958,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12206407636404037,
      "rewards/margins": 0.013219590298831463,
      "rewards/rejected": -0.1352836638689041,
      "sft_loss": 1.220640778541565,
      "step": 1130
    },
    {
      "epoch": 1.9305673158340388,
      "grad_norm": 1.925073504447937,
      "learning_rate": 1.406873743724065e-06,
      "logits/chosen": 254.1175537109375,
      "logits/rejected": 253.83895874023438,
      "logps/chosen": -1.2488387823104858,
      "logps/rejected": -1.625109314918518,
      "loss": 1.3124,
      "odds_ratio_loss": 0.6353055238723755,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.12488389015197754,
      "rewards/margins": 0.03762705251574516,
      "rewards/rejected": -0.1625109314918518,
      "sft_loss": 1.2488387823104858,
      "step": 1140
    },
    {
      "epoch": 1.947502116850127,
      "grad_norm": 1.9894033670425415,
      "learning_rate": 1.3671395808397898e-06,
      "logits/chosen": 254.44320678710938,
      "logits/rejected": 254.1394500732422,
      "logps/chosen": -1.1459629535675049,
      "logps/rejected": -1.3053247928619385,
      "loss": 1.212,
      "odds_ratio_loss": 0.6603742241859436,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.11459628492593765,
      "rewards/margins": 0.015936192125082016,
      "rewards/rejected": -0.13053248822689056,
      "sft_loss": 1.1459629535675049,
      "step": 1150
    },
    {
      "epoch": 1.964436917866215,
      "grad_norm": 8.539448738098145,
      "learning_rate": 1.3277623349995418e-06,
      "logits/chosen": 254.93515014648438,
      "logits/rejected": 256.26190185546875,
      "logps/chosen": -1.2024309635162354,
      "logps/rejected": -1.3554456233978271,
      "loss": 1.2708,
      "odds_ratio_loss": 0.6840348243713379,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12024309486150742,
      "rewards/margins": 0.015301482751965523,
      "rewards/rejected": -0.1355445832014084,
      "sft_loss": 1.2024309635162354,
      "step": 1160
    },
    {
      "epoch": 1.9813717188823032,
      "grad_norm": 3.0065865516662598,
      "learning_rate": 1.2887544123302781e-06,
      "logits/chosen": 255.7893524169922,
      "logits/rejected": 255.7041473388672,
      "logps/chosen": -1.2803890705108643,
      "logps/rejected": -1.3648675680160522,
      "loss": 1.3554,
      "odds_ratio_loss": 0.7498189210891724,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.1280389130115509,
      "rewards/margins": 0.008447853848338127,
      "rewards/rejected": -0.1364867389202118,
      "sft_loss": 1.2803890705108643,
      "step": 1170
    },
    {
      "epoch": 1.9983065198983911,
      "grad_norm": 1.9976747035980225,
      "learning_rate": 1.2501281026006393e-06,
      "logits/chosen": 253.8057403564453,
      "logits/rejected": 253.51309204101562,
      "logps/chosen": -1.2094168663024902,
      "logps/rejected": -1.379926323890686,
      "loss": 1.2795,
      "odds_ratio_loss": 0.7010444402694702,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.12094169855117798,
      "rewards/margins": 0.017050940543413162,
      "rewards/rejected": -0.13799263536930084,
      "sft_loss": 1.2094168663024902,
      "step": 1180
    },
    {
      "epoch": 2.015241320914479,
      "grad_norm": 2.3396496772766113,
      "learning_rate": 1.2118955753489523e-06,
      "logits/chosen": 253.90792846679688,
      "logits/rejected": 254.0242462158203,
      "logps/chosen": -1.2840607166290283,
      "logps/rejected": -1.4544405937194824,
      "loss": 1.3511,
      "odds_ratio_loss": 0.6707578897476196,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.1284060776233673,
      "rewards/margins": 0.01703798957169056,
      "rewards/rejected": -0.14544406533241272,
      "sft_loss": 1.2840607166290283,
      "step": 1190
    },
    {
      "epoch": 2.032176121930567,
      "grad_norm": 3.239753007888794,
      "learning_rate": 1.1740688760491189e-06,
      "logits/chosen": 253.80667114257812,
      "logits/rejected": 254.23452758789062,
      "logps/chosen": -1.284844160079956,
      "logps/rejected": -1.5715402364730835,
      "loss": 1.3467,
      "odds_ratio_loss": 0.6180799007415771,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12848441302776337,
      "rewards/margins": 0.02866961434483528,
      "rewards/rejected": -0.15715403854846954,
      "sft_loss": 1.284844160079956,
      "step": 1200
    },
    {
      "epoch": 2.0491109229466553,
      "grad_norm": 3.396362066268921,
      "learning_rate": 1.1366599223155847e-06,
      "logits/chosen": 254.4167022705078,
      "logits/rejected": 254.499755859375,
      "logps/chosen": -1.2031781673431396,
      "logps/rejected": -1.4534327983856201,
      "loss": 1.2695,
      "odds_ratio_loss": 0.66279536485672,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12031783163547516,
      "rewards/margins": 0.025025445967912674,
      "rewards/rejected": -0.14534327387809753,
      "sft_loss": 1.2031781673431396,
      "step": 1210
    },
    {
      "epoch": 2.0660457239627434,
      "grad_norm": 2.935955762863159,
      "learning_rate": 1.0996805001486067e-06,
      "logits/chosen": 253.8230438232422,
      "logits/rejected": 254.1538848876953,
      "logps/chosen": -1.2398602962493896,
      "logps/rejected": -1.4281715154647827,
      "loss": 1.306,
      "odds_ratio_loss": 0.6616999506950378,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12398604303598404,
      "rewards/margins": 0.018831118941307068,
      "rewards/rejected": -0.1428171545267105,
      "sft_loss": 1.2398602962493896,
      "step": 1220
    },
    {
      "epoch": 2.0829805249788316,
      "grad_norm": 5.044885635375977,
      "learning_rate": 1.0631422602209608e-06,
      "logits/chosen": 253.986083984375,
      "logits/rejected": 254.6449737548828,
      "logps/chosen": -1.2787022590637207,
      "logps/rejected": -1.4583269357681274,
      "loss": 1.3487,
      "odds_ratio_loss": 0.700279951095581,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12787023186683655,
      "rewards/margins": 0.01796245574951172,
      "rewards/rejected": -0.14583268761634827,
      "sft_loss": 1.2787022590637207,
      "step": 1230
    },
    {
      "epoch": 2.0999153259949197,
      "grad_norm": 1.7120074033737183,
      "learning_rate": 1.027056714207319e-06,
      "logits/chosen": 255.4936981201172,
      "logits/rejected": 255.05712890625,
      "logps/chosen": -1.317291498184204,
      "logps/rejected": -1.599304437637329,
      "loss": 1.3792,
      "odds_ratio_loss": 0.6188714504241943,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1317291557788849,
      "rewards/margins": 0.02820129692554474,
      "rewards/rejected": -0.15993043780326843,
      "sft_loss": 1.317291498184204,
      "step": 1240
    },
    {
      "epoch": 2.116850127011008,
      "grad_norm": 2.1434166431427,
      "learning_rate": 9.914352311573838e-07,
      "logits/chosen": 254.13870239257812,
      "logits/rejected": 253.87496948242188,
      "logps/chosen": -1.0639550685882568,
      "logps/rejected": -1.3396233320236206,
      "loss": 1.1253,
      "odds_ratio_loss": 0.6130257248878479,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.10639550536870956,
      "rewards/margins": 0.027566824108362198,
      "rewards/rejected": -0.13396233320236206,
      "sft_loss": 1.0639550685882568,
      "step": 1250
    },
    {
      "epoch": 2.1337849280270955,
      "grad_norm": 1.7394886016845703,
      "learning_rate": 9.562890339139877e-07,
      "logits/chosen": 252.9702606201172,
      "logits/rejected": 253.41909790039062,
      "logps/chosen": -1.1834205389022827,
      "logps/rejected": -1.4220540523529053,
      "loss": 1.2498,
      "odds_ratio_loss": 0.6642176508903503,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11834204196929932,
      "rewards/margins": 0.02386336401104927,
      "rewards/rejected": -0.14220541715621948,
      "sft_loss": 1.1834205389022827,
      "step": 1260
    },
    {
      "epoch": 2.1507197290431836,
      "grad_norm": 2.333282232284546,
      "learning_rate": 9.216291955772374e-07,
      "logits/chosen": 253.9619903564453,
      "logits/rejected": 254.09951782226562,
      "logps/chosen": -1.1872704029083252,
      "logps/rejected": -1.3576862812042236,
      "loss": 1.2522,
      "odds_ratio_loss": 0.6492857933044434,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.11872704327106476,
      "rewards/margins": 0.017041588202118874,
      "rewards/rejected": -0.13576863706111908,
      "sft_loss": 1.1872704029083252,
      "step": 1270
    },
    {
      "epoch": 2.167654530059272,
      "grad_norm": 6.051000595092773,
      "learning_rate": 8.874666360158457e-07,
      "logits/chosen": 253.09268188476562,
      "logits/rejected": 253.14846801757812,
      "logps/chosen": -1.1666978597640991,
      "logps/rejected": -1.3771264553070068,
      "loss": 1.2333,
      "odds_ratio_loss": 0.6663509607315063,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.11666979640722275,
      "rewards/margins": 0.021042857319116592,
      "rewards/rejected": -0.13771264255046844,
      "sft_loss": 1.1666978597640991,
      "step": 1280
    },
    {
      "epoch": 2.18458933107536,
      "grad_norm": 2.681307315826416,
      "learning_rate": 8.538121184267315e-07,
      "logits/chosen": 252.548095703125,
      "logits/rejected": 253.38436889648438,
      "logps/chosen": -1.107893705368042,
      "logps/rejected": -1.3506486415863037,
      "loss": 1.1702,
      "odds_ratio_loss": 0.6233555674552917,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11078937351703644,
      "rewards/margins": 0.024275511503219604,
      "rewards/rejected": -0.13506487011909485,
      "sft_loss": 1.107893705368042,
      "step": 1290
    },
    {
      "epoch": 2.201524132091448,
      "grad_norm": 2.337139368057251,
      "learning_rate": 8.206762459439907e-07,
      "logits/chosen": 252.81088256835938,
      "logits/rejected": 253.55819702148438,
      "logps/chosen": -1.2137644290924072,
      "logps/rejected": -1.405731201171875,
      "loss": 1.2819,
      "odds_ratio_loss": 0.6810044050216675,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12137645483016968,
      "rewards/margins": 0.019196683540940285,
      "rewards/rejected": -0.14057312905788422,
      "sft_loss": 1.2137644290924072,
      "step": 1300
    },
    {
      "epoch": 2.218458933107536,
      "grad_norm": 2.8654181957244873,
      "learning_rate": 7.880694582982898e-07,
      "logits/chosen": 254.5794677734375,
      "logits/rejected": 255.022705078125,
      "logps/chosen": -1.2548917531967163,
      "logps/rejected": -1.4934711456298828,
      "loss": 1.3177,
      "odds_ratio_loss": 0.6284439563751221,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12548920512199402,
      "rewards/margins": 0.02385791949927807,
      "rewards/rejected": -0.14934709668159485,
      "sft_loss": 1.2548917531967163,
      "step": 1310
    },
    {
      "epoch": 2.235393734123624,
      "grad_norm": 2.6947178840637207,
      "learning_rate": 7.560020285277401e-07,
      "logits/chosen": 254.29129028320312,
      "logits/rejected": 254.16165161132812,
      "logps/chosen": -1.2197682857513428,
      "logps/rejected": -1.3262290954589844,
      "loss": 1.2905,
      "odds_ratio_loss": 0.7075908780097961,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.1219768151640892,
      "rewards/margins": 0.010646088980138302,
      "rewards/rejected": -0.13262291252613068,
      "sft_loss": 1.2197682857513428,
      "step": 1320
    },
    {
      "epoch": 2.252328535139712,
      "grad_norm": 3.4201550483703613,
      "learning_rate": 7.244840597412956e-07,
      "logits/chosen": 253.959228515625,
      "logits/rejected": 254.095458984375,
      "logps/chosen": -1.244969129562378,
      "logps/rejected": -1.3216217756271362,
      "loss": 1.3196,
      "odds_ratio_loss": 0.7459138631820679,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12449691444635391,
      "rewards/margins": 0.007665269076824188,
      "rewards/rejected": -0.1321621835231781,
      "sft_loss": 1.244969129562378,
      "step": 1330
    },
    {
      "epoch": 2.2692633361558,
      "grad_norm": 4.195887565612793,
      "learning_rate": 6.935254819356796e-07,
      "logits/chosen": 255.4742889404297,
      "logits/rejected": 255.7603759765625,
      "logps/chosen": -1.221064805984497,
      "logps/rejected": -1.4360755681991577,
      "loss": 1.2872,
      "odds_ratio_loss": 0.6609319448471069,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12210647761821747,
      "rewards/margins": 0.021501056849956512,
      "rewards/rejected": -0.14360754191875458,
      "sft_loss": 1.221064805984497,
      "step": 1340
    },
    {
      "epoch": 2.2861981371718882,
      "grad_norm": 1.8310470581054688,
      "learning_rate": 6.631360488668662e-07,
      "logits/chosen": 253.05575561523438,
      "logits/rejected": 253.70724487304688,
      "logps/chosen": -1.1173431873321533,
      "logps/rejected": -1.3836314678192139,
      "loss": 1.1808,
      "odds_ratio_loss": 0.634362518787384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11173433065414429,
      "rewards/margins": 0.02662881650030613,
      "rewards/rejected": -0.13836315274238586,
      "sft_loss": 1.1173431873321533,
      "step": 1350
    },
    {
      "epoch": 2.3031329381879764,
      "grad_norm": 3.754059076309204,
      "learning_rate": 6.333253349770672e-07,
      "logits/chosen": 255.578125,
      "logits/rejected": 255.3465118408203,
      "logps/chosen": -1.257102370262146,
      "logps/rejected": -1.4100966453552246,
      "loss": 1.3274,
      "odds_ratio_loss": 0.7030817270278931,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12571023404598236,
      "rewards/margins": 0.015299415215849876,
      "rewards/rejected": -0.14100965857505798,
      "sft_loss": 1.257102370262146,
      "step": 1360
    },
    {
      "epoch": 2.3200677392040645,
      "grad_norm": 1.6480987071990967,
      "learning_rate": 6.041027323782364e-07,
      "logits/chosen": 254.044921875,
      "logits/rejected": 254.229248046875,
      "logps/chosen": -1.1556494235992432,
      "logps/rejected": -1.4046926498413086,
      "loss": 1.2175,
      "odds_ratio_loss": 0.6188673973083496,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.11556495726108551,
      "rewards/margins": 0.02490430511534214,
      "rewards/rejected": -0.1404692679643631,
      "sft_loss": 1.1556494235992432,
      "step": 1370
    },
    {
      "epoch": 2.337002540220152,
      "grad_norm": 2.606630563735962,
      "learning_rate": 5.754774478929969e-07,
      "logits/chosen": 253.4460906982422,
      "logits/rejected": 253.7098388671875,
      "logps/chosen": -1.2188177108764648,
      "logps/rejected": -1.42665433883667,
      "loss": 1.2822,
      "odds_ratio_loss": 0.6334772706031799,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12188176810741425,
      "rewards/margins": 0.020783668383955956,
      "rewards/rejected": -0.14266543090343475,
      "sft_loss": 1.2188177108764648,
      "step": 1380
    },
    {
      "epoch": 2.3539373412362403,
      "grad_norm": 4.384988784790039,
      "learning_rate": 5.474585001539634e-07,
      "logits/chosen": 254.74624633789062,
      "logits/rejected": 255.39364624023438,
      "logps/chosen": -1.1506468057632446,
      "logps/rejected": -1.3720335960388184,
      "loss": 1.2119,
      "odds_ratio_loss": 0.612425684928894,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.11506466567516327,
      "rewards/margins": 0.02213868871331215,
      "rewards/rejected": -0.1372033655643463,
      "sft_loss": 1.1506468057632446,
      "step": 1390
    },
    {
      "epoch": 2.3708721422523285,
      "grad_norm": 1.8550916910171509,
      "learning_rate": 5.200547167623424e-07,
      "logits/chosen": 255.02920532226562,
      "logits/rejected": 254.79867553710938,
      "logps/chosen": -1.2350322008132935,
      "logps/rejected": -1.4950242042541504,
      "loss": 1.2975,
      "odds_ratio_loss": 0.6242570281028748,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.12350322306156158,
      "rewards/margins": 0.02599920891225338,
      "rewards/rejected": -0.14950242638587952,
      "sft_loss": 1.2350322008132935,
      "step": 1400
    },
    {
      "epoch": 2.3878069432684166,
      "grad_norm": 1.3218871355056763,
      "learning_rate": 4.932747315067271e-07,
      "logits/chosen": 253.67959594726562,
      "logits/rejected": 253.9455108642578,
      "logps/chosen": -1.1884933710098267,
      "logps/rejected": -1.4974465370178223,
      "loss": 1.2494,
      "odds_ratio_loss": 0.6091898083686829,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.11884935200214386,
      "rewards/margins": 0.030895307660102844,
      "rewards/rejected": -0.1497446447610855,
      "sft_loss": 1.1884933710098267,
      "step": 1410
    },
    {
      "epoch": 2.4047417442845047,
      "grad_norm": 4.312244415283203,
      "learning_rate": 4.6712698164294553e-07,
      "logits/chosen": 253.365966796875,
      "logits/rejected": 254.00637817382812,
      "logps/chosen": -1.1741528511047363,
      "logps/rejected": -1.393843650817871,
      "loss": 1.2374,
      "odds_ratio_loss": 0.6321112513542175,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.11741527169942856,
      "rewards/margins": 0.021969076246023178,
      "rewards/rejected": -0.13938435912132263,
      "sft_loss": 1.1741528511047363,
      "step": 1420
    },
    {
      "epoch": 2.421676545300593,
      "grad_norm": 2.516547918319702,
      "learning_rate": 4.41619705235842e-07,
      "logits/chosen": 254.66812133789062,
      "logits/rejected": 255.44058227539062,
      "logps/chosen": -1.243293285369873,
      "logps/rejected": -1.6239792108535767,
      "loss": 1.3027,
      "odds_ratio_loss": 0.5943632125854492,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.1243293434381485,
      "rewards/margins": 0.03806859627366066,
      "rewards/rejected": -0.16239793598651886,
      "sft_loss": 1.243293285369873,
      "step": 1430
    },
    {
      "epoch": 2.438611346316681,
      "grad_norm": 1.4834553003311157,
      "learning_rate": 4.167609385637961e-07,
      "logits/chosen": 254.78579711914062,
      "logits/rejected": 255.18881225585938,
      "logps/chosen": -1.2608935832977295,
      "logps/rejected": -1.425330638885498,
      "loss": 1.3255,
      "odds_ratio_loss": 0.6463701128959656,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12608936429023743,
      "rewards/margins": 0.016443699598312378,
      "rewards/rejected": -0.1425330489873886,
      "sft_loss": 1.2608935832977295,
      "step": 1440
    },
    {
      "epoch": 2.4555461473327687,
      "grad_norm": 3.3783388137817383,
      "learning_rate": 3.9255851358683567e-07,
      "logits/chosen": 254.845458984375,
      "logits/rejected": 255.02957153320312,
      "logps/chosen": -1.1708389520645142,
      "logps/rejected": -1.3822017908096313,
      "loss": 1.2391,
      "odds_ratio_loss": 0.6828811764717102,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.11708390712738037,
      "rewards/margins": 0.02113628014922142,
      "rewards/rejected": -0.1382201761007309,
      "sft_loss": 1.1708389520645142,
      "step": 1450
    },
    {
      "epoch": 2.472480948348857,
      "grad_norm": 3.0601887702941895,
      "learning_rate": 3.690200554791082e-07,
      "logits/chosen": 252.8287811279297,
      "logits/rejected": 252.99655151367188,
      "logps/chosen": -1.207948923110962,
      "logps/rejected": -1.4645483493804932,
      "loss": 1.2682,
      "odds_ratio_loss": 0.6026018261909485,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12079490721225739,
      "rewards/margins": 0.025659915059804916,
      "rewards/rejected": -0.1464548110961914,
      "sft_loss": 1.207948923110962,
      "step": 1460
    },
    {
      "epoch": 2.489415749364945,
      "grad_norm": 4.717799186706543,
      "learning_rate": 3.461529802265079e-07,
      "logits/chosen": 253.954833984375,
      "logits/rejected": 254.8602752685547,
      "logps/chosen": -1.2261638641357422,
      "logps/rejected": -1.3805973529815674,
      "loss": 1.2931,
      "odds_ratio_loss": 0.6691168546676636,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12261638790369034,
      "rewards/margins": 0.01544335950165987,
      "rewards/rejected": -0.13805975019931793,
      "sft_loss": 1.2261638641357422,
      "step": 1470
    },
    {
      "epoch": 2.506350550381033,
      "grad_norm": 2.0962188243865967,
      "learning_rate": 3.2396449229020883e-07,
      "logits/chosen": 254.4154815673828,
      "logits/rejected": 254.2891082763672,
      "logps/chosen": -1.242130994796753,
      "logps/rejected": -1.4029905796051025,
      "loss": 1.3096,
      "odds_ratio_loss": 0.6747825741767883,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1242130845785141,
      "rewards/margins": 0.016085978597402573,
      "rewards/rejected": -0.14029906690120697,
      "sft_loss": 1.242130994796753,
      "step": 1480
    },
    {
      "epoch": 2.523285351397121,
      "grad_norm": 4.5739922523498535,
      "learning_rate": 3.024615823368371e-07,
      "logits/chosen": 253.8447723388672,
      "logits/rejected": 253.74185180664062,
      "logps/chosen": -1.1965720653533936,
      "logps/rejected": -1.408341407775879,
      "loss": 1.2608,
      "odds_ratio_loss": 0.6427493691444397,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11965718120336533,
      "rewards/margins": 0.021176962181925774,
      "rewards/rejected": -0.14083415269851685,
      "sft_loss": 1.1965720653533936,
      "step": 1490
    },
    {
      "epoch": 2.5402201524132093,
      "grad_norm": 2.1523611545562744,
      "learning_rate": 2.8165102503600716e-07,
      "logits/chosen": 255.3013153076172,
      "logits/rejected": 254.82162475585938,
      "logps/chosen": -1.1430598497390747,
      "logps/rejected": -1.4227603673934937,
      "loss": 1.2057,
      "odds_ratio_loss": 0.6268513798713684,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.11430598795413971,
      "rewards/margins": 0.027970051392912865,
      "rewards/rejected": -0.14227603375911713,
      "sft_loss": 1.1430598497390747,
      "step": 1500
    },
    {
      "epoch": 2.5402201524132093,
      "eval_logits/chosen": 253.5438995361328,
      "eval_logits/rejected": 253.86448669433594,
      "eval_logps/chosen": -1.2487919330596924,
      "eval_logps/rejected": -1.470909833908081,
      "eval_loss": 1.3159173727035522,
      "eval_odds_ratio_loss": 0.6712530851364136,
      "eval_rewards/accuracies": 0.561904788017273,
      "eval_rewards/chosen": -0.12487921118736267,
      "eval_rewards/margins": 0.02221176214516163,
      "eval_rewards/rejected": -0.14709095656871796,
      "eval_runtime": 222.3485,
      "eval_samples_per_second": 4.722,
      "eval_sft_loss": 1.2487919330596924,
      "eval_steps_per_second": 2.361,
      "step": 1500
    },
    {
      "epoch": 2.557154953429297,
      "grad_norm": 1.7336281538009644,
      "learning_rate": 2.615393769259039e-07,
      "logits/chosen": 254.2406768798828,
      "logits/rejected": 254.7086639404297,
      "logps/chosen": -1.319853663444519,
      "logps/rejected": -1.414665937423706,
      "loss": 1.3937,
      "odds_ratio_loss": 0.7381461262702942,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1319853961467743,
      "rewards/margins": 0.009481215849518776,
      "rewards/rejected": -0.14146658778190613,
      "sft_loss": 1.319853663444519,
      "step": 1510
    },
    {
      "epoch": 2.574089754445385,
      "grad_norm": 2.717740297317505,
      "learning_rate": 2.421329743475917e-07,
      "logits/chosen": 254.42990112304688,
      "logits/rejected": 254.5959014892578,
      "logps/chosen": -1.1657871007919312,
      "logps/rejected": -1.3640342950820923,
      "loss": 1.2334,
      "odds_ratio_loss": 0.6763022541999817,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.11657872051000595,
      "rewards/margins": 0.019824711605906487,
      "rewards/rejected": -0.13640344142913818,
      "sft_loss": 1.1657871007919312,
      "step": 1520
    },
    {
      "epoch": 2.5910245554614733,
      "grad_norm": 2.8658461570739746,
      "learning_rate": 2.234379314486973e-07,
      "logits/chosen": 253.9791259765625,
      "logits/rejected": 254.4918670654297,
      "logps/chosen": -1.2597442865371704,
      "logps/rejected": -1.434729814529419,
      "loss": 1.3281,
      "odds_ratio_loss": 0.6833871006965637,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12597441673278809,
      "rewards/margins": 0.017498571425676346,
      "rewards/rejected": -0.14347299933433533,
      "sft_loss": 1.2597442865371704,
      "step": 1530
    },
    {
      "epoch": 2.6079593564775614,
      "grad_norm": 1.626412034034729,
      "learning_rate": 2.0546013825709783e-07,
      "logits/chosen": 254.2110595703125,
      "logits/rejected": 254.682373046875,
      "logps/chosen": -1.2033625841140747,
      "logps/rejected": -1.5869375467300415,
      "loss": 1.2628,
      "odds_ratio_loss": 0.5944391489028931,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12033627182245255,
      "rewards/margins": 0.03835749253630638,
      "rewards/rejected": -0.15869374573230743,
      "sft_loss": 1.2033625841140747,
      "step": 1540
    },
    {
      "epoch": 2.6248941574936495,
      "grad_norm": 3.6966705322265625,
      "learning_rate": 1.88205258825217e-07,
      "logits/chosen": 253.447998046875,
      "logits/rejected": 254.12582397460938,
      "logps/chosen": -1.0821640491485596,
      "logps/rejected": -1.3699367046356201,
      "loss": 1.1414,
      "odds_ratio_loss": 0.5919198393821716,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.10821641981601715,
      "rewards/margins": 0.028777271509170532,
      "rewards/rejected": -0.1369936764240265,
      "sft_loss": 1.0821640491485596,
      "step": 1550
    },
    {
      "epoch": 2.6418289585097376,
      "grad_norm": 2.906128168106079,
      "learning_rate": 1.7167872944552245e-07,
      "logits/chosen": 253.732666015625,
      "logits/rejected": 254.1513214111328,
      "logps/chosen": -1.1799228191375732,
      "logps/rejected": -1.3946034908294678,
      "loss": 1.2452,
      "odds_ratio_loss": 0.6522972583770752,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11799228191375732,
      "rewards/margins": 0.021468069404363632,
      "rewards/rejected": -0.13946035504341125,
      "sft_loss": 1.1799228191375732,
      "step": 1560
    },
    {
      "epoch": 2.6587637595258258,
      "grad_norm": 2.2223029136657715,
      "learning_rate": 1.5588575693777142e-07,
      "logits/chosen": 255.2970428466797,
      "logits/rejected": 255.1865692138672,
      "logps/chosen": -1.1618893146514893,
      "logps/rejected": -1.3099621534347534,
      "loss": 1.2281,
      "odds_ratio_loss": 0.6616276502609253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11618892103433609,
      "rewards/margins": 0.01480729877948761,
      "rewards/rejected": -0.1309962272644043,
      "sft_loss": 1.1618893146514893,
      "step": 1570
    },
    {
      "epoch": 2.675698560541914,
      "grad_norm": 3.071002721786499,
      "learning_rate": 1.4083131700856428e-07,
      "logits/chosen": 252.95462036132812,
      "logits/rejected": 253.45449829101562,
      "logps/chosen": -1.275651454925537,
      "logps/rejected": -1.421471357345581,
      "loss": 1.3435,
      "odds_ratio_loss": 0.6788088083267212,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.1275651454925537,
      "rewards/margins": 0.01458200253546238,
      "rewards/rejected": -0.14214713871479034,
      "sft_loss": 1.275651454925537,
      "step": 1580
    },
    {
      "epoch": 2.6926333615580016,
      "grad_norm": 3.7156240940093994,
      "learning_rate": 1.2652015268370315e-07,
      "logits/chosen": 255.25674438476562,
      "logits/rejected": 255.61105346679688,
      "logps/chosen": -1.173892617225647,
      "logps/rejected": -1.431540846824646,
      "loss": 1.2371,
      "odds_ratio_loss": 0.6318042874336243,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.1173892617225647,
      "rewards/margins": 0.02576482854783535,
      "rewards/rejected": -0.1431540995836258,
      "sft_loss": 1.173892617225647,
      "step": 1590
    },
    {
      "epoch": 2.7095681625740897,
      "grad_norm": 3.5745534896850586,
      "learning_rate": 1.1295677281386502e-07,
      "logits/chosen": 254.6566162109375,
      "logits/rejected": 254.93807983398438,
      "logps/chosen": -1.3123610019683838,
      "logps/rejected": -1.6006231307983398,
      "loss": 1.3777,
      "odds_ratio_loss": 0.6533895134925842,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.13123610615730286,
      "rewards/margins": 0.028826218098402023,
      "rewards/rejected": -0.16006234288215637,
      "sft_loss": 1.3123610019683838,
      "step": 1600
    },
    {
      "epoch": 2.726502963590178,
      "grad_norm": 2.5153346061706543,
      "learning_rate": 1.0014545065404973e-07,
      "logits/chosen": 254.83837890625,
      "logits/rejected": 255.0562286376953,
      "logps/chosen": -1.2541964054107666,
      "logps/rejected": -1.501185655593872,
      "loss": 1.323,
      "odds_ratio_loss": 0.6881270408630371,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12541964650154114,
      "rewards/margins": 0.02469891868531704,
      "rewards/rejected": -0.15011855959892273,
      "sft_loss": 1.2541964054107666,
      "step": 1610
    },
    {
      "epoch": 2.743437764606266,
      "grad_norm": 1.9238418340682983,
      "learning_rate": 8.809022251725502e-08,
      "logits/chosen": 255.2646484375,
      "logits/rejected": 255.28665161132812,
      "logps/chosen": -1.2217843532562256,
      "logps/rejected": -1.5161769390106201,
      "loss": 1.285,
      "odds_ratio_loss": 0.6321364045143127,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12217842042446136,
      "rewards/margins": 0.029439279809594154,
      "rewards/rejected": -0.15161770582199097,
      "sft_loss": 1.2217843532562256,
      "step": 1620
    },
    {
      "epoch": 2.7603725656223537,
      "grad_norm": 2.2195630073547363,
      "learning_rate": 7.679488650280509e-08,
      "logits/chosen": 256.6839904785156,
      "logits/rejected": 256.9764404296875,
      "logps/chosen": -1.246552586555481,
      "logps/rejected": -1.428499460220337,
      "loss": 1.3117,
      "odds_ratio_loss": 0.6510922312736511,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12465526908636093,
      "rewards/margins": 0.01819469965994358,
      "rewards/rejected": -0.14284996688365936,
      "sft_loss": 1.246552586555481,
      "step": 1630
    },
    {
      "epoch": 2.777307366638442,
      "grad_norm": 5.003395080566406,
      "learning_rate": 6.626300129972563e-08,
      "logits/chosen": 254.61721801757812,
      "logits/rejected": 255.3614044189453,
      "logps/chosen": -1.1782416105270386,
      "logps/rejected": -1.39430570602417,
      "loss": 1.2449,
      "odds_ratio_loss": 0.6665213704109192,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.11782417446374893,
      "rewards/margins": 0.021606411784887314,
      "rewards/rejected": -0.13943056762218475,
      "sft_loss": 1.1782416105270386,
      "step": 1640
    },
    {
      "epoch": 2.79424216765453,
      "grad_norm": 2.193207263946533,
      "learning_rate": 5.649788506555065e-08,
      "logits/chosen": 253.5979461669922,
      "logits/rejected": 254.72201538085938,
      "logps/chosen": -1.2255361080169678,
      "logps/rejected": -1.4795286655426025,
      "loss": 1.2878,
      "odds_ratio_loss": 0.6227026581764221,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12255360931158066,
      "rewards/margins": 0.02539927139878273,
      "rewards/rejected": -0.1479528844356537,
      "sft_loss": 1.2255361080169678,
      "step": 1650
    },
    {
      "epoch": 2.811176968670618,
      "grad_norm": 2.1340768337249756,
      "learning_rate": 4.7502614380908474e-08,
      "logits/chosen": 255.0865020751953,
      "logits/rejected": 255.60958862304688,
      "logps/chosen": -1.1954472064971924,
      "logps/rejected": -1.39599609375,
      "loss": 1.2631,
      "odds_ratio_loss": 0.6766428351402283,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.11954472959041595,
      "rewards/margins": 0.020054878666996956,
      "rewards/rejected": -0.13959960639476776,
      "sft_loss": 1.1954472064971924,
      "step": 1660
    },
    {
      "epoch": 2.828111769686706,
      "grad_norm": 2.8715949058532715,
      "learning_rate": 3.9280023280222066e-08,
      "logits/chosen": 254.49807739257812,
      "logits/rejected": 254.8501434326172,
      "logps/chosen": -1.1896106004714966,
      "logps/rejected": -1.3948595523834229,
      "loss": 1.2531,
      "odds_ratio_loss": 0.6348463892936707,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11896105855703354,
      "rewards/margins": 0.020524896681308746,
      "rewards/rejected": -0.1394859403371811,
      "sft_loss": 1.1896106004714966,
      "step": 1670
    },
    {
      "epoch": 2.8450465707027943,
      "grad_norm": 1.6025958061218262,
      "learning_rate": 3.1832702358818855e-08,
      "logits/chosen": 254.23095703125,
      "logits/rejected": 254.564208984375,
      "logps/chosen": -1.2220954895019531,
      "logps/rejected": -1.390905737876892,
      "loss": 1.288,
      "odds_ratio_loss": 0.6586533784866333,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12220954895019531,
      "rewards/margins": 0.016881009563803673,
      "rewards/rejected": -0.13909055292606354,
      "sft_loss": 1.2220954895019531,
      "step": 1680
    },
    {
      "epoch": 2.8619813717188824,
      "grad_norm": 2.6114532947540283,
      "learning_rate": 2.5162997956746647e-08,
      "logits/chosen": 254.39810180664062,
      "logits/rejected": 254.410400390625,
      "logps/chosen": -1.2105869054794312,
      "logps/rejected": -1.523001790046692,
      "loss": 1.2743,
      "odds_ratio_loss": 0.6369892954826355,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.12105868011713028,
      "rewards/margins": 0.031241485849022865,
      "rewards/rejected": -0.15230019390583038,
      "sft_loss": 1.2105869054794312,
      "step": 1690
    },
    {
      "epoch": 2.8789161727349706,
      "grad_norm": 5.513568878173828,
      "learning_rate": 1.9273011419536914e-08,
      "logits/chosen": 254.4501495361328,
      "logits/rejected": 254.602783203125,
      "logps/chosen": -1.2663971185684204,
      "logps/rejected": -1.3631950616836548,
      "loss": 1.3412,
      "odds_ratio_loss": 0.7475703954696655,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1266397088766098,
      "rewards/margins": 0.009679802693426609,
      "rewards/rejected": -0.13631951808929443,
      "sft_loss": 1.2663971185684204,
      "step": 1700
    },
    {
      "epoch": 2.8958509737510583,
      "grad_norm": 2.4077532291412354,
      "learning_rate": 1.4164598436159083e-08,
      "logits/chosen": 254.43917846679688,
      "logits/rejected": 254.9999237060547,
      "logps/chosen": -1.2015140056610107,
      "logps/rejected": -1.3471964597702026,
      "loss": 1.2679,
      "odds_ratio_loss": 0.6641986966133118,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12015138566493988,
      "rewards/margins": 0.014568252488970757,
      "rewards/rejected": -0.1347196400165558,
      "sft_loss": 1.2015140056610107,
      "step": 1710
    },
    {
      "epoch": 2.9127857747671464,
      "grad_norm": 1.6812494993209839,
      "learning_rate": 9.839368454371556e-09,
      "logits/chosen": 253.5565185546875,
      "logits/rejected": 253.43716430664062,
      "logps/chosen": -1.1501060724258423,
      "logps/rejected": -1.454131007194519,
      "loss": 1.213,
      "odds_ratio_loss": 0.6291440725326538,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.11501061916351318,
      "rewards/margins": 0.03040250577032566,
      "rewards/rejected": -0.1454131305217743,
      "sft_loss": 1.1501060724258423,
      "step": 1720
    },
    {
      "epoch": 2.9297205757832345,
      "grad_norm": 1.9758036136627197,
      "learning_rate": 6.298684173650649e-09,
      "logits/chosen": 255.69775390625,
      "logits/rejected": 255.80801391601562,
      "logps/chosen": -1.1473584175109863,
      "logps/rejected": -1.4585325717926025,
      "loss": 1.2129,
      "odds_ratio_loss": 0.6559008359909058,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.11473584175109863,
      "rewards/margins": 0.031117429956793785,
      "rewards/rejected": -0.14585328102111816,
      "sft_loss": 1.1473584175109863,
      "step": 1730
    },
    {
      "epoch": 2.9466553767993227,
      "grad_norm": 1.9825971126556396,
      "learning_rate": 3.543661115860686e-09,
      "logits/chosen": 253.49093627929688,
      "logits/rejected": 254.39404296875,
      "logps/chosen": -1.2223292589187622,
      "logps/rejected": -1.468933343887329,
      "loss": 1.2883,
      "odds_ratio_loss": 0.660078763961792,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12223292887210846,
      "rewards/margins": 0.02466042898595333,
      "rewards/rejected": -0.14689335227012634,
      "sft_loss": 1.2223292589187622,
      "step": 1740
    },
    {
      "epoch": 2.963590177815411,
      "grad_norm": 2.3436944484710693,
      "learning_rate": 1.575167273800693e-09,
      "logits/chosen": 253.98019409179688,
      "logits/rejected": 254.1809844970703,
      "logps/chosen": -1.1822068691253662,
      "logps/rejected": -1.3398927450180054,
      "loss": 1.249,
      "odds_ratio_loss": 0.6677161455154419,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11822070181369781,
      "rewards/margins": 0.015768591314554214,
      "rewards/rejected": -0.13398927450180054,
      "sft_loss": 1.1822068691253662,
      "step": 1750
    },
    {
      "epoch": 2.9805249788314985,
      "grad_norm": 6.531856536865234,
      "learning_rate": 3.9382283773564676e-10,
      "logits/chosen": 254.8570098876953,
      "logits/rejected": 255.1466064453125,
      "logps/chosen": -1.272533655166626,
      "logps/rejected": -1.498098611831665,
      "loss": 1.3394,
      "odds_ratio_loss": 0.6682497262954712,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12725338339805603,
      "rewards/margins": 0.02255650982260704,
      "rewards/rejected": -0.14980986714363098,
      "sft_loss": 1.272533655166626,
      "step": 1760
    },
    {
      "epoch": 2.9974597798475866,
      "grad_norm": 10.32663345336914,
      "learning_rate": 0.0,
      "logits/chosen": 253.9761199951172,
      "logits/rejected": 253.9351806640625,
      "logps/chosen": -1.3296254873275757,
      "logps/rejected": -1.604189157485962,
      "loss": 1.3966,
      "odds_ratio_loss": 0.6696831583976746,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.13296253979206085,
      "rewards/margins": 0.02745637856423855,
      "rewards/rejected": -0.16041892766952515,
      "sft_loss": 1.3296254873275757,
      "step": 1770
    },
    {
      "epoch": 2.9974597798475866,
      "step": 1770,
      "total_flos": 2.135641671600046e+18,
      "train_loss": 1.3953795637788071,
      "train_runtime": 23493.5825,
      "train_samples_per_second": 1.206,
      "train_steps_per_second": 0.075
    }
  ],
  "logging_steps": 10,
  "max_steps": 1770,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 2.135641671600046e+18,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}