{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 73,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "grad_norm": 31.510919537856985,
      "learning_rate": 6.25e-08,
      "logps/chosen": -49.24467849731445,
      "logps/rejected": -39.578514099121094,
      "loss": 0.7008,
      "losses/dpo": 0.6596390604972839,
      "losses/sft": 0.17909570038318634,
      "losses/total": 0.6596390604972839,
      "ref_logps/chosen": -49.112640380859375,
      "ref_logps/rejected": -39.590065002441406,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": -0.013204257935285568,
      "rewards/margins": -0.01435948722064495,
      "rewards/rejected": 0.001155228354036808,
      "step": 1
    },
    {
      "epoch": 0.03,
      "grad_norm": 50.71439940508435,
      "learning_rate": 1.25e-07,
      "logps/chosen": -53.134193420410156,
      "logps/rejected": -47.03993225097656,
      "loss": 0.6899,
      "losses/dpo": 0.6913646459579468,
      "losses/sft": 0.1710137277841568,
      "losses/total": 0.6913646459579468,
      "ref_logps/chosen": -53.1795768737793,
      "ref_logps/rejected": -47.00617599487305,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": 0.004538209177553654,
      "rewards/margins": 0.007913690991699696,
      "rewards/rejected": -0.0033754808828234673,
      "step": 2
    },
    {
      "epoch": 0.04,
      "grad_norm": 42.035900231625114,
      "learning_rate": 1.875e-07,
      "logps/chosen": -51.67204666137695,
      "logps/rejected": -42.43119812011719,
      "loss": 0.6908,
      "losses/dpo": 0.6752724647521973,
      "losses/sft": 0.1361423283815384,
      "losses/total": 0.6752724647521973,
      "ref_logps/chosen": -51.709320068359375,
      "ref_logps/rejected": -42.41019058227539,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": 0.0037269049789756536,
      "rewards/margins": 0.00582832982763648,
      "rewards/rejected": -0.002101423917338252,
      "step": 3
    },
    {
      "epoch": 0.05,
      "grad_norm": 37.529489917527485,
      "learning_rate": 2.5e-07,
      "logps/chosen": -49.27662658691406,
      "logps/rejected": -42.45417404174805,
      "loss": 0.6949,
      "losses/dpo": 0.6901852488517761,
      "losses/sft": 0.18635118007659912,
      "losses/total": 0.6901852488517761,
      "ref_logps/chosen": -49.26988220214844,
      "ref_logps/rejected": -42.47301483154297,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0006745874416083097,
      "rewards/margins": -0.0025582504458725452,
      "rewards/rejected": 0.0018836627714335918,
      "step": 4
    },
    {
      "epoch": 0.07,
      "grad_norm": 33.99395990426859,
      "learning_rate": 3.1249999999999997e-07,
      "logps/chosen": -48.10963439941406,
      "logps/rejected": -40.53562927246094,
      "loss": 0.693,
      "losses/dpo": 0.6474289894104004,
      "losses/sft": 0.1237928494811058,
      "losses/total": 0.6474289894104004,
      "ref_logps/chosen": -48.120365142822266,
      "ref_logps/rejected": -40.53252410888672,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 0.001073037856258452,
      "rewards/margins": 0.001383528346195817,
      "rewards/rejected": -0.00031049124663695693,
      "step": 5
    },
    {
      "epoch": 0.08,
      "grad_norm": 33.57616672063881,
      "learning_rate": 3.75e-07,
      "logps/chosen": -50.0238037109375,
      "logps/rejected": -41.26359558105469,
      "loss": 0.6976,
      "losses/dpo": 0.6670458316802979,
      "losses/sft": 0.15494827926158905,
      "losses/total": 0.6670458316802979,
      "ref_logps/chosen": -49.96715545654297,
      "ref_logps/rejected": -41.28278732299805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005665063392370939,
      "rewards/margins": -0.007583992555737495,
      "rewards/rejected": 0.0019189291633665562,
      "step": 6
    },
    {
      "epoch": 0.1,
      "grad_norm": 32.82497056854739,
      "learning_rate": 4.375e-07,
      "logps/chosen": -51.68867111206055,
      "logps/rejected": -43.37660217285156,
      "loss": 0.6921,
      "losses/dpo": 0.6866487264633179,
      "losses/sft": 0.13379694521427155,
      "losses/total": 0.6866487264633179,
      "ref_logps/chosen": -51.77002716064453,
      "ref_logps/rejected": -43.42585754394531,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.008136047050356865,
      "rewards/margins": 0.0032104291021823883,
      "rewards/rejected": 0.004925617948174477,
      "step": 7
    },
    {
      "epoch": 0.11,
      "grad_norm": 38.79691230760384,
      "learning_rate": 5e-07,
      "logps/chosen": -49.79241180419922,
      "logps/rejected": -42.46006774902344,
      "loss": 0.695,
      "losses/dpo": 0.6527105569839478,
      "losses/sft": 0.15517215430736542,
      "losses/total": 0.6527105569839478,
      "ref_logps/chosen": -49.76615905761719,
      "ref_logps/rejected": -42.45863342285156,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -0.0026245731860399246,
      "rewards/margins": -0.002481258474290371,
      "rewards/rejected": -0.00014331366401165724,
      "step": 8
    },
    {
      "epoch": 0.12,
      "grad_norm": 226.56518786863586,
      "learning_rate": 4.997080567080816e-07,
      "logps/chosen": -54.879547119140625,
      "logps/rejected": -42.98774719238281,
      "loss": 0.6853,
      "losses/dpo": 0.6064831614494324,
      "losses/sft": 0.2268030345439911,
      "losses/total": 0.6064831614494324,
      "ref_logps/chosen": -54.92854309082031,
      "ref_logps/rejected": -42.86476135253906,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": 0.004900116473436356,
      "rewards/margins": 0.017198245972394943,
      "rewards/rejected": -0.012298129498958588,
      "step": 9
    },
    {
      "epoch": 0.14,
      "grad_norm": 50.10962021770019,
      "learning_rate": 4.988329086794122e-07,
      "logps/chosen": -52.08982467651367,
      "logps/rejected": -43.10820388793945,
      "loss": 0.6755,
      "losses/dpo": 0.7082348465919495,
      "losses/sft": 0.1494147777557373,
      "losses/total": 0.7082348465919495,
      "ref_logps/chosen": -52.199188232421875,
      "ref_logps/rejected": -42.8469123840332,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 0.010936222970485687,
      "rewards/margins": 0.0370652973651886,
      "rewards/rejected": -0.02612907439470291,
      "step": 10
    },
    {
      "epoch": 0.15,
      "grad_norm": 50.96166024464587,
      "learning_rate": 4.973765998627628e-07,
      "logps/chosen": -47.0179443359375,
      "logps/rejected": -42.876380920410156,
      "loss": 0.6764,
      "losses/dpo": 0.7454954385757446,
      "losses/sft": 0.11485376209020615,
      "losses/total": 0.7454954385757446,
      "ref_logps/chosen": -47.048309326171875,
      "ref_logps/rejected": -42.54939651489258,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.003036007285118103,
      "rewards/margins": 0.03573445603251457,
      "rewards/rejected": -0.03269844502210617,
      "step": 11
    },
    {
      "epoch": 0.16,
      "grad_norm": 43.95886877722781,
      "learning_rate": 4.953425315348533e-07,
      "logps/chosen": -47.39234161376953,
      "logps/rejected": -42.44198226928711,
      "loss": 0.6802,
      "losses/dpo": 0.6931471824645996,
      "losses/sft": 0.1578434258699417,
      "losses/total": 0.6931471824645996,
      "ref_logps/chosen": -47.38432312011719,
      "ref_logps/rejected": -42.15151596069336,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": -0.0008019641973078251,
      "rewards/margins": 0.028244582936167717,
      "rewards/rejected": -0.029046546667814255,
      "step": 12
    },
    {
      "epoch": 0.18,
      "grad_norm": 35.92010395880078,
      "learning_rate": 4.92735454356513e-07,
      "logps/chosen": -47.71138381958008,
      "logps/rejected": -40.651554107666016,
      "loss": 0.6671,
      "losses/dpo": 0.6601188778877258,
      "losses/sft": 0.12200119346380234,
      "losses/total": 0.6601188778877258,
      "ref_logps/chosen": -47.742637634277344,
      "ref_logps/rejected": -40.12622833251953,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.003125616116449237,
      "rewards/margins": 0.055658161640167236,
      "rewards/rejected": -0.052532538771629333,
      "step": 13
    },
    {
      "epoch": 0.19,
      "grad_norm": 38.42448037262164,
      "learning_rate": 4.895614572772916e-07,
      "logps/chosen": -49.50225830078125,
      "logps/rejected": -41.507652282714844,
      "loss": 0.6657,
      "losses/dpo": 0.7294200658798218,
      "losses/sft": 0.15328460931777954,
      "losses/total": 0.7294200658798218,
      "ref_logps/chosen": -49.43043518066406,
      "ref_logps/rejected": -40.846160888671875,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.007182045839726925,
      "rewards/margins": 0.058967407792806625,
      "rewards/rejected": -0.06614945828914642,
      "step": 14
    },
    {
      "epoch": 0.21,
      "grad_norm": 35.51673879153569,
      "learning_rate": 4.858279533144357e-07,
      "logps/chosen": -50.58116149902344,
      "logps/rejected": -44.15226745605469,
      "loss": 0.6648,
      "losses/dpo": 0.7195499539375305,
      "losses/sft": 0.12852783501148224,
      "losses/total": 0.7195499539375305,
      "ref_logps/chosen": -50.35996627807617,
      "ref_logps/rejected": -43.29161071777344,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.02211933210492134,
      "rewards/margins": 0.06394598633050919,
      "rewards/rejected": -0.08606532216072083,
      "step": 15
    },
    {
      "epoch": 0.22,
      "grad_norm": 34.20405799205637,
      "learning_rate": 4.815436622394441e-07,
      "logps/chosen": -53.629356384277344,
      "logps/rejected": -45.9289665222168,
      "loss": 0.6455,
      "losses/dpo": 0.6931471824645996,
      "losses/sft": 0.14978539943695068,
      "losses/total": 0.6931471824645996,
      "ref_logps/chosen": -53.546119689941406,
      "ref_logps/rejected": -44.75611114501953,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.00832361076027155,
      "rewards/margins": 0.10896164178848267,
      "rewards/rejected": -0.11728526651859283,
      "step": 16
    },
    {
      "epoch": 0.23,
      "grad_norm": 51.47866505976329,
      "learning_rate": 4.767185902126363e-07,
      "logps/chosen": -52.55246353149414,
      "logps/rejected": -48.530906677246094,
      "loss": 0.6445,
      "losses/dpo": 0.6025716662406921,
      "losses/sft": 0.17564229667186737,
      "losses/total": 0.6025716662406921,
      "ref_logps/chosen": -52.30644989013672,
      "ref_logps/rejected": -47.189701080322266,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.02460121549665928,
      "rewards/margins": 0.10951894521713257,
      "rewards/rejected": -0.1341201663017273,
      "step": 17
    },
    {
      "epoch": 0.25,
      "grad_norm": 42.49951077408584,
      "learning_rate": 4.7136400641330245e-07,
      "logps/chosen": -52.193199157714844,
      "logps/rejected": -43.489959716796875,
      "loss": 0.6225,
      "losses/dpo": 0.4434754252433777,
      "losses/sft": 0.13364653289318085,
      "losses/total": 0.4434754252433777,
      "ref_logps/chosen": -51.82938003540039,
      "ref_logps/rejected": -41.477630615234375,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.03638196364045143,
      "rewards/margins": 0.16485059261322021,
      "rewards/rejected": -0.20123255252838135,
      "step": 18
    },
    {
      "epoch": 0.26,
      "grad_norm": 34.30863345836372,
      "learning_rate": 4.6549241672001225e-07,
      "logps/chosen": -49.611228942871094,
      "logps/rejected": -46.756103515625,
      "loss": 0.6071,
      "losses/dpo": 0.6045699715614319,
      "losses/sft": 0.13909928500652313,
      "losses/total": 0.6045699715614319,
      "ref_logps/chosen": -49.14532470703125,
      "ref_logps/rejected": -44.29924774169922,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.046590566635131836,
      "rewards/margins": 0.19909515976905823,
      "rewards/rejected": -0.24568572640419006,
      "step": 19
    },
    {
      "epoch": 0.27,
      "grad_norm": 35.537459009303014,
      "learning_rate": 4.591175345025566e-07,
      "logps/chosen": -50.094032287597656,
      "logps/rejected": -46.121707916259766,
      "loss": 0.6058,
      "losses/dpo": 0.5356044173240662,
      "losses/sft": 0.1512974053621292,
      "losses/total": 0.5356044173240662,
      "ref_logps/chosen": -49.32575607299805,
      "ref_logps/rejected": -43.3276481628418,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.07682754099369049,
      "rewards/margins": 0.20257848501205444,
      "rewards/rejected": -0.27940604090690613,
      "step": 20
    },
    {
      "epoch": 0.29,
      "grad_norm": 82.79188209516856,
      "learning_rate": 4.5225424859373684e-07,
      "logps/chosen": -53.73488235473633,
      "logps/rejected": -48.036128997802734,
      "loss": 0.5936,
      "losses/dpo": 0.6363024711608887,
      "losses/sft": 0.18370339274406433,
      "losses/total": 0.6363024711608887,
      "ref_logps/chosen": -52.835113525390625,
      "ref_logps/rejected": -44.653133392333984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08997656404972076,
      "rewards/margins": 0.24832308292388916,
      "rewards/rejected": -0.33829963207244873,
      "step": 21
    },
    {
      "epoch": 0.3,
      "grad_norm": 35.68500953981497,
      "learning_rate": 4.4491858851580553e-07,
      "logps/chosen": -52.28797149658203,
      "logps/rejected": -48.59185791015625,
      "loss": 0.5956,
      "losses/dpo": 0.6950743198394775,
      "losses/sft": 0.3061237335205078,
      "losses/total": 0.6950743198394775,
      "ref_logps/chosen": -50.30400085449219,
      "ref_logps/rejected": -43.93888854980469,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": -0.19839699566364288,
      "rewards/margins": 0.2669002413749695,
      "rewards/rejected": -0.46529728174209595,
      "step": 22
    },
    {
      "epoch": 0.32,
      "grad_norm": 51.2013048213769,
      "learning_rate": 4.3712768704277524e-07,
      "logps/chosen": -50.60584259033203,
      "logps/rejected": -49.11386489868164,
      "loss": 0.5413,
      "losses/dpo": 1.0638338327407837,
      "losses/sft": 0.1654026359319687,
      "losses/total": 1.0638338327407837,
      "ref_logps/chosen": -48.48411560058594,
      "ref_logps/rejected": -42.86372375488281,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.2121725082397461,
      "rewards/margins": 0.4128417372703552,
      "rewards/rejected": -0.6250141859054565,
      "step": 23
    },
    {
      "epoch": 0.33,
      "grad_norm": 26.152258770479694,
      "learning_rate": 4.2889974018603024e-07,
      "logps/chosen": -53.90440368652344,
      "logps/rejected": -50.59031295776367,
      "loss": 0.5421,
      "losses/dpo": 0.7262043952941895,
      "losses/sft": 0.18153545260429382,
      "losses/total": 0.7262043952941895,
      "ref_logps/chosen": -51.315208435058594,
      "ref_logps/rejected": -43.73783874511719,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.2589200735092163,
      "rewards/margins": 0.4263269305229187,
      "rewards/rejected": -0.6852469444274902,
      "step": 24
    },
    {
      "epoch": 0.34,
      "grad_norm": 49.33026538547341,
      "learning_rate": 4.2025396469669926e-07,
      "logps/chosen": -53.82769775390625,
      "logps/rejected": -50.70613098144531,
      "loss": 0.5517,
      "losses/dpo": 0.41891151666641235,
      "losses/sft": 0.1567077487707138,
      "losses/total": 0.41891151666641235,
      "ref_logps/chosen": -50.04922103881836,
      "ref_logps/rejected": -42.47876739501953,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.37784749269485474,
      "rewards/margins": 0.4448884427547455,
      "rewards/rejected": -0.8227359056472778,
      "step": 25
    },
    {
      "epoch": 0.36,
      "grad_norm": 38.54013817065724,
      "learning_rate": 4.112105531840426e-07,
      "logps/chosen": -56.83380889892578,
      "logps/rejected": -55.79877471923828,
      "loss": 0.5145,
      "losses/dpo": 0.25203394889831543,
      "losses/sft": 0.15007992088794708,
      "losses/total": 0.25203394889831543,
      "ref_logps/chosen": -51.92417907714844,
      "ref_logps/rejected": -45.255348205566406,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.49096283316612244,
      "rewards/margins": 0.5633795261383057,
      "rewards/rejected": -1.0543423891067505,
      "step": 26
    },
    {
      "epoch": 0.37,
      "grad_norm": 35.25775541354148,
      "learning_rate": 4.017906269546778e-07,
      "logps/chosen": -57.45189666748047,
      "logps/rejected": -53.96529769897461,
      "loss": 0.5329,
      "losses/dpo": 0.3053484857082367,
      "losses/sft": 0.12551482021808624,
      "losses/total": 0.3053484857082367,
      "ref_logps/chosen": -52.189697265625,
      "ref_logps/rejected": -44.003353118896484,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5262198448181152,
      "rewards/margins": 0.46997466683387756,
      "rewards/rejected": -0.99619460105896,
      "step": 27
    },
    {
      "epoch": 0.38,
      "grad_norm": 23.64859158816546,
      "learning_rate": 3.920161866827889e-07,
      "logps/chosen": -57.529579162597656,
      "logps/rejected": -56.24253845214844,
      "loss": 0.4882,
      "losses/dpo": 0.67308509349823,
      "losses/sft": 0.12211936712265015,
      "losses/total": 0.67308509349823,
      "ref_logps/chosen": -51.290321350097656,
      "ref_logps/rejected": -43.70161437988281,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.6239261627197266,
      "rewards/margins": 0.6301661133766174,
      "rewards/rejected": -1.2540922164916992,
      "step": 28
    },
    {
      "epoch": 0.4,
      "grad_norm": 34.075060841718205,
      "learning_rate": 3.8191006102653317e-07,
      "logps/chosen": -54.39567947387695,
      "logps/rejected": -53.69304656982422,
      "loss": 0.515,
      "losses/dpo": 0.3636208176612854,
      "losses/sft": 0.15627621114253998,
      "losses/total": 0.3636208176612854,
      "ref_logps/chosen": -46.61610412597656,
      "ref_logps/rejected": -40.169898986816406,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.7779572010040283,
      "rewards/margins": 0.5743571519851685,
      "rewards/rejected": -1.3523142337799072,
      "step": 29
    },
    {
      "epoch": 0.41,
      "grad_norm": 39.075544491508126,
      "learning_rate": 3.7149585331065145e-07,
      "logps/chosen": -55.852046966552734,
      "logps/rejected": -56.0112419128418,
      "loss": 0.4987,
      "losses/dpo": 0.9463083744049072,
      "losses/sft": 0.1410434991121292,
      "losses/total": 0.9463083744049072,
      "ref_logps/chosen": -48.19220733642578,
      "ref_logps/rejected": -42.009246826171875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.765984058380127,
      "rewards/margins": 0.634215772151947,
      "rewards/rejected": -1.4001998901367188,
      "step": 30
    },
    {
      "epoch": 0.42,
      "grad_norm": 27.598875905640913,
      "learning_rate": 3.6079788639981036e-07,
      "logps/chosen": -60.89286804199219,
      "logps/rejected": -58.91151428222656,
      "loss": 0.477,
      "losses/dpo": 0.5266605615615845,
      "losses/sft": 0.14658664166927338,
      "losses/total": 0.5266605615615845,
      "ref_logps/chosen": -51.61863327026367,
      "ref_logps/rejected": -42.28534698486328,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.9274235963821411,
      "rewards/margins": 0.7351932525634766,
      "rewards/rejected": -1.6626169681549072,
      "step": 31
    },
    {
      "epoch": 0.44,
      "grad_norm": 28.756497878941868,
      "learning_rate": 3.498411458914238e-07,
      "logps/chosen": -60.40979766845703,
      "logps/rejected": -60.34819793701172,
      "loss": 0.4847,
      "losses/dpo": 0.4884541630744934,
      "losses/sft": 0.19494454562664032,
      "losses/total": 0.4884541630744934,
      "ref_logps/chosen": -52.04460906982422,
      "ref_logps/rejected": -44.97260665893555,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8365185260772705,
      "rewards/margins": 0.701040506362915,
      "rewards/rejected": -1.537559151649475,
      "step": 32
    },
    {
      "epoch": 0.45,
      "grad_norm": 28.434577214059978,
      "learning_rate": 3.3865122176063385e-07,
      "logps/chosen": -57.96968460083008,
      "logps/rejected": -56.83502960205078,
      "loss": 0.4677,
      "losses/dpo": 0.5375417470932007,
      "losses/sft": 0.148137629032135,
      "losses/total": 0.5375417470932007,
      "ref_logps/chosen": -47.51142120361328,
      "ref_logps/rejected": -38.619873046875,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.0458261966705322,
      "rewards/margins": 0.7756892442703247,
      "rewards/rejected": -1.8215153217315674,
      "step": 33
    },
    {
      "epoch": 0.47,
      "grad_norm": 29.49134519608509,
      "learning_rate": 3.272542485937368e-07,
      "logps/chosen": -62.14280700683594,
      "logps/rejected": -61.54853820800781,
      "loss": 0.4961,
      "losses/dpo": 0.7278311848640442,
      "losses/sft": 0.1993408501148224,
      "losses/total": 0.7278311848640442,
      "ref_logps/chosen": -51.82103729248047,
      "ref_logps/rejected": -44.025291442871094,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0321770906448364,
      "rewards/margins": 0.7201482653617859,
      "rewards/rejected": -1.7523252964019775,
      "step": 34
    },
    {
      "epoch": 0.48,
      "grad_norm": 26.468466564434074,
      "learning_rate": 3.1567684454964674e-07,
      "logps/chosen": -56.682411193847656,
      "logps/rejected": -58.0302734375,
      "loss": 0.4396,
      "losses/dpo": 0.5649744868278503,
      "losses/sft": 0.129312202334404,
      "losses/total": 0.5649744868278503,
      "ref_logps/chosen": -47.770896911621094,
      "ref_logps/rejected": -40.46588897705078,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8911517858505249,
      "rewards/margins": 0.8652870059013367,
      "rewards/rejected": -1.7564387321472168,
      "step": 35
    },
    {
      "epoch": 0.49,
      "grad_norm": 25.595938355510107,
      "learning_rate": 3.0394604919195157e-07,
      "logps/chosen": -61.79121398925781,
      "logps/rejected": -62.89993667602539,
      "loss": 0.4745,
      "losses/dpo": 0.2738940417766571,
      "losses/sft": 0.15838046371936798,
      "losses/total": 0.2738940417766571,
      "ref_logps/chosen": -49.21980285644531,
      "ref_logps/rejected": -42.15373992919922,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -1.257140874862671,
      "rewards/margins": 0.8174787759780884,
      "rewards/rejected": -2.074619770050049,
      "step": 36
    },
    {
      "epoch": 0.51,
      "grad_norm": 24.43093018755541,
      "learning_rate": 2.920892603367596e-07,
      "logps/chosen": -60.27898406982422,
      "logps/rejected": -61.00114822387695,
      "loss": 0.4791,
      "losses/dpo": 0.5097879767417908,
      "losses/sft": 0.17959001660346985,
      "losses/total": 0.5097879767417908,
      "ref_logps/chosen": -49.49871826171875,
      "ref_logps/rejected": -42.78453063964844,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -1.078026533126831,
      "rewards/margins": 0.7436352372169495,
      "rewards/rejected": -1.8216617107391357,
      "step": 37
    },
    {
      "epoch": 0.52,
      "grad_norm": 27.08759050745191,
      "learning_rate": 2.801341700638307e-07,
      "logps/chosen": -62.44047164916992,
      "logps/rejected": -62.224884033203125,
      "loss": 0.4767,
      "losses/dpo": 0.37803658843040466,
      "losses/sft": 0.18652431666851044,
      "losses/total": 0.37803658843040466,
      "ref_logps/chosen": -50.557559967041016,
      "ref_logps/rejected": -41.98173522949219,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.1882915496826172,
      "rewards/margins": 0.8360234498977661,
      "rewards/rejected": -2.0243148803710938,
      "step": 38
    },
    {
      "epoch": 0.53,
      "grad_norm": 33.43704122805882,
      "learning_rate": 2.681087000404406e-07,
      "logps/chosen": -65.03423309326172,
      "logps/rejected": -65.04768371582031,
      "loss": 0.5084,
      "losses/dpo": 0.7027170658111572,
      "losses/sft": 0.12946006655693054,
      "losses/total": 0.7027170658111572,
      "ref_logps/chosen": -51.817020416259766,
      "ref_logps/rejected": -44.10376739501953,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3217215538024902,
      "rewards/margins": 0.7726694345474243,
      "rewards/rejected": -2.094391107559204,
      "step": 39
    },
    {
      "epoch": 0.55,
      "grad_norm": 129.45570096870608,
      "learning_rate": 2.5604093630903305e-07,
      "logps/chosen": -59.94195556640625,
      "logps/rejected": -63.19525909423828,
      "loss": 0.4527,
      "losses/dpo": 0.7064068913459778,
      "losses/sft": 0.13724827766418457,
      "losses/total": 0.7064068913459778,
      "ref_logps/chosen": -47.65526580810547,
      "ref_logps/rejected": -42.33349609375,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.2286689281463623,
      "rewards/margins": 0.8575077056884766,
      "rewards/rejected": -2.086176872253418,
      "step": 40
    },
    {
      "epoch": 0.56,
      "grad_norm": 24.012368893482737,
      "learning_rate": 2.43959063690967e-07,
      "logps/chosen": -61.86077880859375,
      "logps/rejected": -66.92259979248047,
      "loss": 0.4553,
      "losses/dpo": 0.3154614567756653,
      "losses/sft": 0.16294528543949127,
      "losses/total": 0.3154614567756653,
      "ref_logps/chosen": -48.4053955078125,
      "ref_logps/rejected": -44.16739273071289,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.3455381393432617,
      "rewards/margins": 0.9299829006195068,
      "rewards/rejected": -2.2755210399627686,
      "step": 41
    },
    {
      "epoch": 0.58,
      "grad_norm": 102.16273879292123,
      "learning_rate": 2.3189129995955942e-07,
      "logps/chosen": -64.28011322021484,
      "logps/rejected": -64.86600494384766,
      "loss": 0.4642,
      "losses/dpo": 0.4425658583641052,
      "losses/sft": 0.11157938092947006,
      "losses/total": 0.4425658583641052,
      "ref_logps/chosen": -53.148860931396484,
      "ref_logps/rejected": -45.69209289550781,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.113125205039978,
      "rewards/margins": 0.8042662143707275,
      "rewards/rejected": -1.9173914194107056,
      "step": 42
    },
    {
      "epoch": 0.59,
      "grad_norm": 27.45890404880792,
      "learning_rate": 2.1986582993616925e-07,
      "logps/chosen": -64.73760223388672,
      "logps/rejected": -63.79928207397461,
      "loss": 0.4607,
      "losses/dpo": 0.08850211650133133,
      "losses/sft": 0.1770356148481369,
      "losses/total": 0.08850211650133133,
      "ref_logps/chosen": -52.7093391418457,
      "ref_logps/rejected": -42.18277359008789,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.2028264999389648,
      "rewards/margins": 0.9588243961334229,
      "rewards/rejected": -2.1616508960723877,
      "step": 43
    },
    {
      "epoch": 0.6,
      "grad_norm": 35.977504794321945,
      "learning_rate": 2.0791073966324034e-07,
      "logps/chosen": -65.26310729980469,
      "logps/rejected": -66.70973205566406,
      "loss": 0.4531,
      "losses/dpo": 0.7271055579185486,
      "losses/sft": 0.1396849900484085,
      "losses/total": 0.7271055579185486,
      "ref_logps/chosen": -53.6609001159668,
      "ref_logps/rejected": -46.01152801513672,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -1.1602208614349365,
      "rewards/margins": 0.909599781036377,
      "rewards/rejected": -2.0698208808898926,
      "step": 44
    },
    {
      "epoch": 0.62,
      "grad_norm": 30.8655762680209,
      "learning_rate": 1.960539508080485e-07,
      "logps/chosen": -62.69697952270508,
      "logps/rejected": -64.87743377685547,
      "loss": 0.4482,
      "losses/dpo": 0.7160241603851318,
      "losses/sft": 0.16245654225349426,
      "losses/total": 0.7160241603851318,
      "ref_logps/chosen": -49.983802795410156,
      "ref_logps/rejected": -43.17237091064453,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.2713178396224976,
      "rewards/margins": 0.8991883993148804,
      "rewards/rejected": -2.170506238937378,
      "step": 45
    },
    {
      "epoch": 0.63,
      "grad_norm": 42.203067350847334,
      "learning_rate": 1.8432315545035327e-07,
      "logps/chosen": -65.14309692382812,
      "logps/rejected": -64.79584503173828,
      "loss": 0.4638,
      "losses/dpo": 0.647499144077301,
      "losses/sft": 0.17309847474098206,
      "losses/total": 0.647499144077301,
      "ref_logps/chosen": -51.99322509765625,
      "ref_logps/rejected": -42.78443145751953,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -1.3149867057800293,
      "rewards/margins": 0.8861545324325562,
      "rewards/rejected": -2.201141357421875,
      "step": 46
    },
    {
      "epoch": 0.64,
      "grad_norm": 26.70333995315738,
      "learning_rate": 1.7274575140626315e-07,
      "logps/chosen": -64.62788391113281,
      "logps/rejected": -66.00929260253906,
      "loss": 0.3755,
      "losses/dpo": 0.7572497129440308,
      "losses/sft": 0.16420619189739227,
      "losses/total": 0.7572497129440308,
      "ref_logps/chosen": -53.670265197753906,
      "ref_logps/rejected": -43.973365783691406,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -1.0957624912261963,
      "rewards/margins": 1.1078301668167114,
      "rewards/rejected": -2.203592538833618,
      "step": 47
    },
    {
      "epoch": 0.66,
      "grad_norm": 30.007336064883145,
      "learning_rate": 1.6134877823936607e-07,
      "logps/chosen": -60.07398223876953,
      "logps/rejected": -63.938331604003906,
      "loss": 0.4222,
      "losses/dpo": 0.4377414286136627,
      "losses/sft": 0.11289513111114502,
      "losses/total": 0.4377414286136627,
      "ref_logps/chosen": -48.226722717285156,
      "ref_logps/rejected": -42.14054870605469,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.1847256422042847,
      "rewards/margins": 0.995052695274353,
      "rewards/rejected": -2.1797783374786377,
      "step": 48
    },
    {
      "epoch": 0.67,
      "grad_norm": 33.18269229637362,
      "learning_rate": 1.5015885410857614e-07,
      "logps/chosen": -63.292476654052734,
      "logps/rejected": -66.98123168945312,
      "loss": 0.4246,
      "losses/dpo": 0.3581680655479431,
      "losses/sft": 0.17332157492637634,
      "losses/total": 0.3581680655479431,
      "ref_logps/chosen": -50.916664123535156,
      "ref_logps/rejected": -45.03084182739258,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.2375812530517578,
      "rewards/margins": 0.9574571847915649,
      "rewards/rejected": -2.195038318634033,
      "step": 49
    },
    {
      "epoch": 0.68,
      "grad_norm": 32.535671402481796,
      "learning_rate": 1.392021136001897e-07,
      "logps/chosen": -61.653507232666016,
      "logps/rejected": -66.69647979736328,
      "loss": 0.3784,
      "losses/dpo": 0.14524562656879425,
      "losses/sft": 0.1722523272037506,
      "losses/total": 0.14524562656879425,
      "ref_logps/chosen": -51.076236724853516,
      "ref_logps/rejected": -44.246944427490234,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -1.0577270984649658,
      "rewards/margins": 1.1872267723083496,
      "rewards/rejected": -2.2449538707733154,
      "step": 50
    },
    {
      "epoch": 0.7,
      "grad_norm": 25.494002931399418,
      "learning_rate": 1.2850414668934847e-07,
      "logps/chosen": -63.76778030395508,
      "logps/rejected": -66.36669921875,
      "loss": 0.4239,
      "losses/dpo": 0.5600922107696533,
      "losses/sft": 0.13496388494968414,
      "losses/total": 0.5600922107696533,
      "ref_logps/chosen": -49.28026580810547,
      "ref_logps/rejected": -41.501556396484375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.4487512111663818,
      "rewards/margins": 1.0377633571624756,
      "rewards/rejected": -2.4865145683288574,
      "step": 51
    },
    {
      "epoch": 0.71,
      "grad_norm": 24.257143077705184,
      "learning_rate": 1.1808993897346678e-07,
      "logps/chosen": -60.98112487792969,
      "logps/rejected": -66.6905517578125,
      "loss": 0.4463,
      "losses/dpo": 0.6072797775268555,
      "losses/sft": 0.14662015438079834,
      "losses/total": 0.6072797775268555,
      "ref_logps/chosen": -48.362701416015625,
      "ref_logps/rejected": -44.13787841796875,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -1.2618422508239746,
      "rewards/margins": 0.9934254288673401,
      "rewards/rejected": -2.25526762008667,
      "step": 52
    },
    {
      "epoch": 0.73,
      "grad_norm": 28.99400991059714,
      "learning_rate": 1.0798381331721107e-07,
      "logps/chosen": -61.79584884643555,
      "logps/rejected": -64.41800689697266,
      "loss": 0.4073,
      "losses/dpo": 0.17208106815814972,
      "losses/sft": 0.14093177020549774,
      "losses/total": 0.17208106815814972,
      "ref_logps/chosen": -49.47767639160156,
      "ref_logps/rejected": -41.640419006347656,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -1.2318172454833984,
      "rewards/margins": 1.0459411144256592,
      "rewards/rejected": -2.2777583599090576,
      "step": 53
    },
    {
      "epoch": 0.74,
      "grad_norm": 32.15451904929163,
      "learning_rate": 9.82093730453222e-08,
      "logps/chosen": -62.59501647949219,
      "logps/rejected": -63.971126556396484,
      "loss": 0.5058,
      "losses/dpo": 0.3638117015361786,
      "losses/sft": 0.14573359489440918,
      "losses/total": 0.3638117015361786,
      "ref_logps/chosen": -51.01821517944336,
      "ref_logps/rejected": -43.79859161376953,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -1.157679796218872,
      "rewards/margins": 0.8595736026763916,
      "rewards/rejected": -2.0172533988952637,
      "step": 54
    },
    {
      "epoch": 0.75,
      "grad_norm": 29.71547196346147,
      "learning_rate": 8.87894468159574e-08,
      "logps/chosen": -63.908302307128906,
      "logps/rejected": -65.4727554321289,
      "loss": 0.4043,
      "losses/dpo": 1.4689037799835205,
      "losses/sft": 0.19870857894420624,
      "losses/total": 1.4689037799835205,
      "ref_logps/chosen": -49.52888870239258,
      "ref_logps/rejected": -39.608917236328125,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -1.437941312789917,
      "rewards/margins": 1.148442029953003,
      "rewards/rejected": -2.586383581161499,
      "step": 55
    },
    {
      "epoch": 0.77,
      "grad_norm": 40.070212554436814,
      "learning_rate": 7.974603530330067e-08,
      "logps/chosen": -63.663780212402344,
      "logps/rejected": -65.04582977294922,
      "loss": 0.4551,
      "losses/dpo": 0.7427124977111816,
      "losses/sft": 0.1415950208902359,
      "losses/total": 0.7427124977111816,
      "ref_logps/chosen": -52.447853088378906,
      "ref_logps/rejected": -44.403289794921875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.1215929985046387,
      "rewards/margins": 0.9426611661911011,
      "rewards/rejected": -2.0642542839050293,
      "step": 56
    },
    {
      "epoch": 0.78,
      "grad_norm": 50.773025060467745,
      "learning_rate": 7.110025981396975e-08,
      "logps/chosen": -64.57611083984375,
      "logps/rejected": -67.85655212402344,
      "loss": 0.4677,
      "losses/dpo": 0.7480742335319519,
      "losses/sft": 0.1996467560529709,
      "losses/total": 0.7480742335319519,
      "ref_logps/chosen": -49.66741943359375,
      "ref_logps/rejected": -43.6092414855957,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.4908695220947266,
      "rewards/margins": 0.9338614344596863,
      "rewards/rejected": -2.4247307777404785,
      "step": 57
    },
    {
      "epoch": 0.79,
      "grad_norm": 151.98374859438076,
      "learning_rate": 6.28723129572247e-08,
      "logps/chosen": -63.19204330444336,
      "logps/rejected": -67.11865234375,
      "loss": 0.4125,
      "losses/dpo": 0.191142737865448,
      "losses/sft": 0.1258397251367569,
      "losses/total": 0.191142737865448,
      "ref_logps/chosen": -49.3205680847168,
      "ref_logps/rejected": -42.308128356933594,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -1.387147307395935,
      "rewards/margins": 1.0939054489135742,
      "rewards/rejected": -2.481052875518799,
      "step": 58
    },
    {
      "epoch": 0.81,
      "grad_norm": 68.1040384525775,
      "learning_rate": 5.508141148419443e-08,
      "logps/chosen": -66.73199462890625,
      "logps/rejected": -66.16500854492188,
      "loss": 0.416,
      "losses/dpo": 0.4806699752807617,
      "losses/sft": 0.17691649496555328,
      "losses/total": 0.4806699752807617,
      "ref_logps/chosen": -52.95691680908203,
      "ref_logps/rejected": -42.309608459472656,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -1.3775070905685425,
      "rewards/margins": 1.0080331563949585,
      "rewards/rejected": -2.38554048538208,
      "step": 59
    },
    {
      "epoch": 0.82,
      "grad_norm": 40.930870088616274,
      "learning_rate": 4.774575140626316e-08,
      "logps/chosen": -65.13594055175781,
      "logps/rejected": -67.62340545654297,
      "loss": 0.4084,
      "losses/dpo": 0.3780416250228882,
      "losses/sft": 0.070173479616642,
      "losses/total": 0.3780416250228882,
      "ref_logps/chosen": -50.85914611816406,
      "ref_logps/rejected": -41.8093147277832,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -1.4276790618896484,
      "rewards/margins": 1.153730034828186,
      "rewards/rejected": -2.581409215927124,
      "step": 60
    },
    {
      "epoch": 0.84,
      "grad_norm": 30.260562657822298,
      "learning_rate": 4.0882465497443313e-08,
      "logps/chosen": -63.07252883911133,
      "logps/rejected": -69.03398132324219,
      "loss": 0.4369,
      "losses/dpo": 0.36678266525268555,
      "losses/sft": 0.12201398611068726,
      "losses/total": 0.36678266525268555,
      "ref_logps/chosen": -49.61937713623047,
      "ref_logps/rejected": -44.632652282714844,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.3453152179718018,
      "rewards/margins": 1.0948173999786377,
      "rewards/rejected": -2.4401326179504395,
      "step": 61
    },
    {
      "epoch": 0.85,
      "grad_norm": 24.402238672879346,
      "learning_rate": 3.450758327998768e-08,
      "logps/chosen": -63.65578842163086,
      "logps/rejected": -67.62722778320312,
      "loss": 0.3813,
      "losses/dpo": 0.7685420513153076,
      "losses/sft": 0.19448544085025787,
      "losses/total": 0.7685420513153076,
      "ref_logps/chosen": -51.653221130371094,
      "ref_logps/rejected": -43.50916290283203,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -1.2002568244934082,
      "rewards/margins": 1.2115496397018433,
      "rewards/rejected": -2.411806583404541,
      "step": 62
    },
    {
      "epoch": 0.86,
      "grad_norm": 24.77840549005978,
      "learning_rate": 2.863599358669755e-08,
      "logps/chosen": -65.08932495117188,
      "logps/rejected": -66.61253356933594,
      "loss": 0.4291,
      "losses/dpo": 1.248164176940918,
      "losses/sft": 0.15433447062969208,
      "losses/total": 1.248164176940918,
      "ref_logps/chosen": -52.82990646362305,
      "ref_logps/rejected": -44.81475830078125,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -1.2259418964385986,
      "rewards/margins": 0.9538350105285645,
      "rewards/rejected": -2.179776906967163,
      "step": 63
    },
    {
      "epoch": 0.88,
      "grad_norm": 37.580661698465605,
      "learning_rate": 2.3281409787363648e-08,
      "logps/chosen": -61.57237243652344,
      "logps/rejected": -65.20365142822266,
      "loss": 0.4063,
      "losses/dpo": 0.09678905457258224,
      "losses/sft": 0.15513929724693298,
      "losses/total": 0.09678905457258224,
      "ref_logps/chosen": -48.36906814575195,
      "ref_logps/rejected": -41.755638122558594,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.3203303813934326,
      "rewards/margins": 1.0244702100753784,
      "rewards/rejected": -2.3448004722595215,
      "step": 64
    },
    {
      "epoch": 0.89,
      "grad_norm": 52.86928073665909,
      "learning_rate": 1.845633776055591e-08,
      "logps/chosen": -63.794559478759766,
      "logps/rejected": -68.59591674804688,
      "loss": 0.4236,
      "losses/dpo": 0.5401971936225891,
      "losses/sft": 0.1492851972579956,
      "losses/total": 0.5401971936225891,
      "ref_logps/chosen": -51.490814208984375,
      "ref_logps/rejected": -46.14380645751953,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2303749322891235,
      "rewards/margins": 1.0148357152938843,
      "rewards/rejected": -2.245210647583008,
      "step": 65
    },
    {
      "epoch": 0.9,
      "grad_norm": 26.186736903994795,
      "learning_rate": 1.4172046685564209e-08,
      "logps/chosen": -63.65757751464844,
      "logps/rejected": -66.78644561767578,
      "loss": 0.4453,
      "losses/dpo": 0.3751629889011383,
      "losses/sft": 0.20083920657634735,
      "losses/total": 0.3751629889011383,
      "ref_logps/chosen": -50.14271545410156,
      "ref_logps/rejected": -43.08906555175781,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.3514864444732666,
      "rewards/margins": 1.0182515382766724,
      "rewards/rejected": -2.3697378635406494,
      "step": 66
    },
    {
      "epoch": 0.92,
      "grad_norm": 32.512692665801914,
      "learning_rate": 1.0438542722708444e-08,
      "logps/chosen": -61.425235748291016,
      "logps/rejected": -62.99015426635742,
      "loss": 0.4034,
      "losses/dpo": 0.4745466709136963,
      "losses/sft": 0.15217916667461395,
      "losses/total": 0.4745466709136963,
      "ref_logps/chosen": -49.22258758544922,
      "ref_logps/rejected": -39.8096923828125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.2202647924423218,
      "rewards/margins": 1.0977811813354492,
      "rewards/rejected": -2.3180460929870605,
      "step": 67
    },
    {
      "epoch": 0.93,
      "grad_norm": 28.142561520937964,
      "learning_rate": 7.2645456434869965e-09,
      "logps/chosen": -60.124473571777344,
      "logps/rejected": -64.52591705322266,
      "loss": 0.4073,
      "losses/dpo": 0.5824078917503357,
      "losses/sft": 0.15901198983192444,
      "losses/total": 0.5824078917503357,
      "ref_logps/chosen": -49.08730697631836,
      "ref_logps/rejected": -42.66169357299805,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -1.1037168502807617,
      "rewards/margins": 1.0827052593231201,
      "rewards/rejected": -2.1864218711853027,
      "step": 68
    },
    {
      "epoch": 0.95,
      "grad_norm": 26.50196447879399,
      "learning_rate": 4.657468465146641e-09,
      "logps/chosen": -63.6911735534668,
      "logps/rejected": -68.59868621826172,
      "loss": 0.3973,
      "losses/dpo": 0.40227654576301575,
      "losses/sft": 0.18853811919689178,
      "losses/total": 0.40227654576301575,
      "ref_logps/chosen": -51.581443786621094,
      "ref_logps/rejected": -44.77821350097656,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -1.210972785949707,
      "rewards/margins": 1.1710745096206665,
      "rewards/rejected": -2.382047176361084,
      "step": 69
    },
    {
      "epoch": 0.96,
      "grad_norm": 56.71695513519511,
      "learning_rate": 2.6234001372372193e-09,
      "logps/chosen": -63.47015380859375,
      "logps/rejected": -65.77841186523438,
      "loss": 0.4462,
      "losses/dpo": 0.6931471824645996,
      "losses/sft": 0.16308167576789856,
      "losses/total": 0.6931471824645996,
      "ref_logps/chosen": -50.47125244140625,
      "ref_logps/rejected": -43.315406799316406,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -1.2998902797698975,
      "rewards/margins": 0.9464105367660522,
      "rewards/rejected": -2.24630069732666,
      "step": 70
    },
    {
      "epoch": 0.97,
      "grad_norm": 23.674451477206517,
      "learning_rate": 1.167091320587843e-09,
      "logps/chosen": -62.78862762451172,
      "logps/rejected": -67.10327911376953,
      "loss": 0.4313,
      "losses/dpo": 0.12619969248771667,
      "losses/sft": 0.16695602238178253,
      "losses/total": 0.12619969248771667,
      "ref_logps/chosen": -50.83470153808594,
      "ref_logps/rejected": -45.14189147949219,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -1.1953924894332886,
      "rewards/margins": 1.000746250152588,
      "rewards/rejected": -2.196138858795166,
      "step": 71
    },
    {
      "epoch": 0.99,
      "grad_norm": 26.52528545158649,
      "learning_rate": 2.9194329191833953e-10,
      "logps/chosen": -63.891353607177734,
      "logps/rejected": -66.84733581542969,
      "loss": 0.4241,
      "losses/dpo": 0.2748945951461792,
      "losses/sft": 0.1392793506383896,
      "losses/total": 0.2748945951461792,
      "ref_logps/chosen": -51.043392181396484,
      "ref_logps/rejected": -43.61123275756836,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -1.2847963571548462,
      "rewards/margins": 1.0388145446777344,
      "rewards/rejected": -2.323610782623291,
      "step": 72
    },
    {
      "epoch": 1.0,
      "grad_norm": 32.933679564175364,
      "learning_rate": 0.0,
      "logps/chosen": -64.90662384033203,
      "logps/rejected": -65.81568145751953,
      "loss": 0.3989,
      "losses/dpo": 0.6386691927909851,
      "losses/sft": 0.17253142595291138,
      "losses/total": 0.6386691927909851,
      "ref_logps/chosen": -53.89427185058594,
      "ref_logps/rejected": -43.687034606933594,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -1.1012351512908936,
      "rewards/margins": 1.1116297245025635,
      "rewards/rejected": -2.212864875793457,
      "step": 73
    },
    {
      "epoch": 1.0,
      "step": 73,
      "total_flos": 0.0,
      "train_loss": 0.5157321454727486,
      "train_runtime": 1200.6075,
      "train_samples_per_second": 7.848,
      "train_steps_per_second": 0.061
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 73,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 20000,
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}