{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 37,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": -3.41206693649292,
      "debug/policy_chosen_logps": -143.21307373046875,
      "debug/policy_rejected_logits": -3.2551214694976807,
      "debug/policy_rejected_logps": -150.2328338623047,
      "debug/reference_chosen_logps": -143.21307373046875,
      "debug/reference_rejected_logps": -150.2328338623047,
      "epoch": 0.02702702702702703,
      "grad_norm": 3.5140838495338116,
      "learning_rate": 1e-06,
      "logits/chosen": -3.41206693649292,
      "logits/rejected": -3.2551214694976807,
      "logps/chosen": -143.21307373046875,
      "logps/rejected": -150.2328338623047,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": -3.4049649238586426,
      "debug/policy_chosen_logps": -143.7670135498047,
      "debug/policy_rejected_logits": -3.4008069038391113,
      "debug/policy_rejected_logps": -138.0754852294922,
      "debug/reference_chosen_logps": -143.82278442382812,
      "debug/reference_rejected_logps": -138.1428985595703,
      "epoch": 0.05405405405405406,
      "grad_norm": 3.526231895119079,
      "learning_rate": 1e-06,
      "logits/chosen": -3.4049649238586426,
      "logits/rejected": -3.4008069038391113,
      "logps/chosen": -143.7670135498047,
      "logps/rejected": -138.0754852294922,
      "loss": 0.4994,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0005577945848926902,
      "rewards/margins": -0.00011638639261946082,
      "rewards/rejected": 0.000674180977512151,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": -3.2883780002593994,
      "debug/policy_chosen_logps": -143.35385131835938,
      "debug/policy_rejected_logits": -3.2103564739227295,
      "debug/policy_rejected_logps": -139.85372924804688,
      "debug/reference_chosen_logps": -143.38311767578125,
      "debug/reference_rejected_logps": -139.98326110839844,
      "epoch": 0.08108108108108109,
      "grad_norm": 3.618148106626628,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2883780002593994,
      "logits/rejected": -3.2103564739227295,
      "logps/chosen": -143.35385131835938,
      "logps/rejected": -139.85372924804688,
      "loss": 0.5006,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.00029258732683956623,
      "rewards/margins": -0.001002683537080884,
      "rewards/rejected": 0.0012952708639204502,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": -3.294776678085327,
      "debug/policy_chosen_logps": -132.79519653320312,
      "debug/policy_rejected_logits": -3.4150912761688232,
      "debug/policy_rejected_logps": -138.08180236816406,
      "debug/reference_chosen_logps": -132.62167358398438,
      "debug/reference_rejected_logps": -137.927978515625,
      "epoch": 0.10810810810810811,
      "grad_norm": 3.5093502617713077,
      "learning_rate": 1e-06,
      "logits/chosen": -3.294776678085327,
      "logits/rejected": -3.4150912761688232,
      "logps/chosen": -132.79519653320312,
      "logps/rejected": -138.08180236816406,
      "loss": 0.4994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0017352199647575617,
      "rewards/margins": -0.0001968955621123314,
      "rewards/rejected": -0.0015383241698145866,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": -3.343012809753418,
      "debug/policy_chosen_logps": -126.95188903808594,
      "debug/policy_rejected_logits": -3.3124771118164062,
      "debug/policy_rejected_logps": -129.9374237060547,
      "debug/reference_chosen_logps": -126.82910919189453,
      "debug/reference_rejected_logps": -129.91493225097656,
      "epoch": 0.13513513513513514,
      "grad_norm": 3.250245901043351,
      "learning_rate": 1e-06,
      "logits/chosen": -3.343012809753418,
      "logits/rejected": -3.3124771118164062,
      "logps/chosen": -126.95188903808594,
      "logps/rejected": -129.9374237060547,
      "loss": 0.5009,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.001227750675752759,
      "rewards/margins": -0.001002778997644782,
      "rewards/rejected": -0.00022497176541946828,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": -3.2374277114868164,
      "debug/policy_chosen_logps": -154.26406860351562,
      "debug/policy_rejected_logits": -3.2978382110595703,
      "debug/policy_rejected_logps": -143.40101623535156,
      "debug/reference_chosen_logps": -153.9961395263672,
      "debug/reference_rejected_logps": -143.3165283203125,
      "epoch": 0.16216216216216217,
      "grad_norm": 3.439711062354016,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2374277114868164,
      "logits/rejected": -3.2978382110595703,
      "logps/chosen": -154.26406860351562,
      "logps/rejected": -143.40101623535156,
      "loss": 0.5009,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002679176162928343,
      "rewards/margins": -0.0018340873066335917,
      "rewards/rejected": -0.000845088972710073,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": -3.2252252101898193,
      "debug/policy_chosen_logps": -162.3474578857422,
      "debug/policy_rejected_logits": -3.230537176132202,
      "debug/policy_rejected_logps": -147.3297576904297,
      "debug/reference_chosen_logps": -162.26889038085938,
      "debug/reference_rejected_logps": -147.57199096679688,
      "epoch": 0.1891891891891892,
      "grad_norm": 3.5128248367150468,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2252252101898193,
      "logits/rejected": -3.230537176132202,
      "logps/chosen": -162.3474578857422,
      "logps/rejected": -147.3297576904297,
      "loss": 0.5005,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0007855510921217501,
      "rewards/margins": -0.003207969479262829,
      "rewards/rejected": 0.0024224184453487396,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": -3.3006181716918945,
      "debug/policy_chosen_logps": -149.18630981445312,
      "debug/policy_rejected_logits": -3.3146488666534424,
      "debug/policy_rejected_logps": -147.72348022460938,
      "debug/reference_chosen_logps": -149.20960998535156,
      "debug/reference_rejected_logps": -147.54893493652344,
      "epoch": 0.21621621621621623,
      "grad_norm": 3.499360817762659,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3006181716918945,
      "logits/rejected": -3.3146488666534424,
      "logps/chosen": -149.18630981445312,
      "logps/rejected": -147.72348022460938,
      "loss": 0.4992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.000233030179515481,
      "rewards/margins": 0.001978444866836071,
      "rewards/rejected": -0.00174541468732059,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": -3.3503048419952393,
      "debug/policy_chosen_logps": -150.99197387695312,
      "debug/policy_rejected_logits": -3.2822837829589844,
      "debug/policy_rejected_logps": -145.26345825195312,
      "debug/reference_chosen_logps": -151.06939697265625,
      "debug/reference_rejected_logps": -145.52816772460938,
      "epoch": 0.24324324324324326,
      "grad_norm": 3.4264003876223215,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3503048419952393,
      "logits/rejected": -3.2822837829589844,
      "logps/chosen": -150.99197387695312,
      "logps/rejected": -145.26345825195312,
      "loss": 0.4987,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0007742022862657905,
      "rewards/margins": -0.0018729494186118245,
      "rewards/rejected": 0.002647151704877615,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": -3.304420232772827,
      "debug/policy_chosen_logps": -133.21044921875,
      "debug/policy_rejected_logits": -3.412870168685913,
      "debug/policy_rejected_logps": -125.90888977050781,
      "debug/reference_chosen_logps": -133.26504516601562,
      "debug/reference_rejected_logps": -125.95523834228516,
      "epoch": 0.2702702702702703,
      "grad_norm": 3.3820605783151074,
      "learning_rate": 1e-06,
      "logits/chosen": -3.304420232772827,
      "logits/rejected": -3.412870168685913,
      "logps/chosen": -133.21044921875,
      "logps/rejected": -125.90888977050781,
      "loss": 0.5005,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0005457591032609344,
      "rewards/margins": 8.22543297545053e-05,
      "rewards/rejected": 0.0004635047516785562,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": -3.3585329055786133,
      "debug/policy_chosen_logps": -139.380859375,
      "debug/policy_rejected_logits": -3.3599538803100586,
      "debug/policy_rejected_logps": -132.3596649169922,
      "debug/reference_chosen_logps": -139.71542358398438,
      "debug/reference_rejected_logps": -132.61981201171875,
      "epoch": 0.2972972972972973,
      "grad_norm": 3.288218575224273,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3585329055786133,
      "logits/rejected": -3.3599538803100586,
      "logps/chosen": -139.380859375,
      "logps/rejected": -132.3596649169922,
      "loss": 0.4999,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0033456420060247183,
      "rewards/margins": 0.0007441901252605021,
      "rewards/rejected": 0.0026014517061412334,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": -3.2882096767425537,
      "debug/policy_chosen_logps": -156.53334045410156,
      "debug/policy_rejected_logits": -3.2960500717163086,
      "debug/policy_rejected_logps": -143.13995361328125,
      "debug/reference_chosen_logps": -156.6325225830078,
      "debug/reference_rejected_logps": -142.83905029296875,
      "epoch": 0.32432432432432434,
      "grad_norm": 3.813316903837083,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2882096767425537,
      "logits/rejected": -3.2960500717163086,
      "logps/chosen": -156.53334045410156,
      "logps/rejected": -143.13995361328125,
      "loss": 0.4989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0009917640127241611,
      "rewards/margins": 0.004000749439001083,
      "rewards/rejected": -0.0030089854262769222,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": -3.2771499156951904,
      "debug/policy_chosen_logps": -133.85403442382812,
      "debug/policy_rejected_logits": -3.3577096462249756,
      "debug/policy_rejected_logps": -145.81051635742188,
      "debug/reference_chosen_logps": -134.21636962890625,
      "debug/reference_rejected_logps": -145.71588134765625,
      "epoch": 0.35135135135135137,
      "grad_norm": 3.5238547026932254,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2771499156951904,
      "logits/rejected": -3.3577096462249756,
      "logps/chosen": -133.85403442382812,
      "logps/rejected": -145.81051635742188,
      "loss": 0.5004,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003623266238719225,
      "rewards/margins": 0.004569797310978174,
      "rewards/rejected": -0.0009465311886742711,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": -3.4804084300994873,
      "debug/policy_chosen_logps": -165.59515380859375,
      "debug/policy_rejected_logits": -3.5207834243774414,
      "debug/policy_rejected_logps": -139.4119873046875,
      "debug/reference_chosen_logps": -165.61618041992188,
      "debug/reference_rejected_logps": -139.546142578125,
      "epoch": 0.3783783783783784,
      "grad_norm": 3.2715325631312377,
      "learning_rate": 1e-06,
      "logits/chosen": -3.4804084300994873,
      "logits/rejected": -3.5207834243774414,
      "logps/chosen": -165.59515380859375,
      "logps/rejected": -139.4119873046875,
      "loss": 0.4997,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00021032325457781553,
      "rewards/margins": -0.0011312393471598625,
      "rewards/rejected": 0.0013415624853223562,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": -3.3723671436309814,
      "debug/policy_chosen_logps": -138.40748596191406,
      "debug/policy_rejected_logits": -3.2726492881774902,
      "debug/policy_rejected_logps": -146.87710571289062,
      "debug/reference_chosen_logps": -138.04966735839844,
      "debug/reference_rejected_logps": -146.4534454345703,
      "epoch": 0.40540540540540543,
      "grad_norm": 3.5479335407438923,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3723671436309814,
      "logits/rejected": -3.2726492881774902,
      "logps/chosen": -138.40748596191406,
      "logps/rejected": -146.87710571289062,
      "loss": 0.4971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003578138304874301,
      "rewards/margins": 0.0006584453512914479,
      "rewards/rejected": -0.004236583597958088,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": -3.280038356781006,
      "debug/policy_chosen_logps": -144.99691772460938,
      "debug/policy_rejected_logits": -3.265355110168457,
      "debug/policy_rejected_logps": -136.1260223388672,
      "debug/reference_chosen_logps": -145.38519287109375,
      "debug/reference_rejected_logps": -136.1790313720703,
      "epoch": 0.43243243243243246,
      "grad_norm": 3.3905692790559057,
      "learning_rate": 1e-06,
      "logits/chosen": -3.280038356781006,
      "logits/rejected": -3.265355110168457,
      "logps/chosen": -144.99691772460938,
      "logps/rejected": -136.1260223388672,
      "loss": 0.4987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0038827608805149794,
      "rewards/margins": 0.0033527181949466467,
      "rewards/rejected": 0.0005300427437759936,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": -3.4201347827911377,
      "debug/policy_chosen_logps": -154.7392578125,
      "debug/policy_rejected_logits": -3.348952293395996,
      "debug/policy_rejected_logps": -133.3173828125,
      "debug/reference_chosen_logps": -154.70895385742188,
      "debug/reference_rejected_logps": -133.4116668701172,
      "epoch": 0.4594594594594595,
      "grad_norm": 3.6801385333166086,
      "learning_rate": 1e-06,
      "logits/chosen": -3.4201347827911377,
      "logits/rejected": -3.348952293395996,
      "logps/chosen": -154.7392578125,
      "logps/rejected": -133.3173828125,
      "loss": 0.4999,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0003031063242815435,
      "rewards/margins": -0.0012458800338208675,
      "rewards/rejected": 0.0009427738841623068,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": -3.2982780933380127,
      "debug/policy_chosen_logps": -146.35263061523438,
      "debug/policy_rejected_logits": -3.3202931880950928,
      "debug/policy_rejected_logps": -156.7283172607422,
      "debug/reference_chosen_logps": -146.34840393066406,
      "debug/reference_rejected_logps": -156.84378051757812,
      "epoch": 0.4864864864864865,
      "grad_norm": 3.5196394680278327,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2982780933380127,
      "logits/rejected": -3.3202931880950928,
      "logps/chosen": -146.35263061523438,
      "logps/rejected": -156.7283172607422,
      "loss": 0.4974,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -4.220963455736637e-05,
      "rewards/margins": -0.0011968993349000812,
      "rewards/rejected": 0.0011546898167580366,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": -3.3398616313934326,
      "debug/policy_chosen_logps": -131.99713134765625,
      "debug/policy_rejected_logits": -3.3604841232299805,
      "debug/policy_rejected_logps": -148.30227661132812,
      "debug/reference_chosen_logps": -131.88931274414062,
      "debug/reference_rejected_logps": -148.13345336914062,
      "epoch": 0.5135135135135135,
      "grad_norm": 3.609790844356815,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3398616313934326,
      "logits/rejected": -3.3604841232299805,
      "logps/chosen": -131.99713134765625,
      "logps/rejected": -148.30227661132812,
      "loss": 0.4981,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0010780429001897573,
      "rewards/margins": 0.0006100271129980683,
      "rewards/rejected": -0.0016880702460184693,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": -3.2643423080444336,
      "debug/policy_chosen_logps": -134.54617309570312,
      "debug/policy_rejected_logits": -3.255173921585083,
      "debug/policy_rejected_logps": -124.35269165039062,
      "debug/reference_chosen_logps": -134.53549194335938,
      "debug/reference_rejected_logps": -123.25413513183594,
      "epoch": 0.5405405405405406,
      "grad_norm": 3.4308218789715035,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2643423080444336,
      "logits/rejected": -3.255173921585083,
      "logps/chosen": -134.54617309570312,
      "logps/rejected": -124.35269165039062,
      "loss": 0.4969,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00010671629570424557,
      "rewards/margins": 0.010878859087824821,
      "rewards/rejected": -0.010985574685037136,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": -3.39021635055542,
      "debug/policy_chosen_logps": -145.20547485351562,
      "debug/policy_rejected_logits": -3.324305295944214,
      "debug/policy_rejected_logps": -140.68399047851562,
      "debug/reference_chosen_logps": -145.19155883789062,
      "debug/reference_rejected_logps": -139.9624481201172,
      "epoch": 0.5675675675675675,
      "grad_norm": 3.572312158938155,
      "learning_rate": 1e-06,
      "logits/chosen": -3.39021635055542,
      "logits/rejected": -3.324305295944214,
      "logps/chosen": -145.20547485351562,
      "logps/rejected": -140.68399047851562,
      "loss": 0.4975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00013918871991336346,
      "rewards/margins": 0.007076186593621969,
      "rewards/rejected": -0.007215375080704689,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": -3.267886161804199,
      "debug/policy_chosen_logps": -160.2618865966797,
      "debug/policy_rejected_logits": -3.2591264247894287,
      "debug/policy_rejected_logps": -141.91876220703125,
      "debug/reference_chosen_logps": -159.80300903320312,
      "debug/reference_rejected_logps": -142.28695678710938,
      "epoch": 0.5945945945945946,
      "grad_norm": 3.81196901542863,
      "learning_rate": 1e-06,
      "logits/chosen": -3.267886161804199,
      "logits/rejected": -3.2591264247894287,
      "logps/chosen": -160.2618865966797,
      "logps/rejected": -141.91876220703125,
      "loss": 0.4995,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.004588775336742401,
      "rewards/margins": -0.008270883932709694,
      "rewards/rejected": 0.0036821081303060055,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": -3.3076066970825195,
      "debug/policy_chosen_logps": -147.66110229492188,
      "debug/policy_rejected_logits": -3.3076066970825195,
      "debug/policy_rejected_logps": -156.1669464111328,
      "debug/reference_chosen_logps": -147.6890869140625,
      "debug/reference_rejected_logps": -155.93359375,
      "epoch": 0.6216216216216216,
      "grad_norm": 3.584513321934443,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3076066970825195,
      "logits/rejected": -3.3076066970825195,
      "logps/chosen": -147.66110229492188,
      "logps/rejected": -156.1669464111328,
      "loss": 0.4981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0002799700014293194,
      "rewards/margins": 0.002613373100757599,
      "rewards/rejected": -0.002333402633666992,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": -3.3157832622528076,
      "debug/policy_chosen_logps": -145.93922424316406,
      "debug/policy_rejected_logits": -3.2806382179260254,
      "debug/policy_rejected_logps": -150.69491577148438,
      "debug/reference_chosen_logps": -146.20852661132812,
      "debug/reference_rejected_logps": -151.0212860107422,
      "epoch": 0.6486486486486487,
      "grad_norm": 3.4973421303279575,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3157832622528076,
      "logits/rejected": -3.2806382179260254,
      "logps/chosen": -145.93922424316406,
      "logps/rejected": -150.69491577148438,
      "loss": 0.4979,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0026929855812340975,
      "rewards/margins": -0.0005707358941435814,
      "rewards/rejected": 0.0032637212425470352,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": -3.340818405151367,
      "debug/policy_chosen_logps": -145.6500244140625,
      "debug/policy_rejected_logits": -3.2608158588409424,
      "debug/policy_rejected_logps": -148.91769409179688,
      "debug/reference_chosen_logps": -145.28700256347656,
      "debug/reference_rejected_logps": -148.38320922851562,
      "epoch": 0.6756756756756757,
      "grad_norm": 3.493835228930821,
      "learning_rate": 1e-06,
      "logits/chosen": -3.340818405151367,
      "logits/rejected": -3.2608158588409424,
      "logps/chosen": -145.6500244140625,
      "logps/rejected": -148.91769409179688,
      "loss": 0.4997,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.003630065592005849,
      "rewards/margins": 0.0017146589234471321,
      "rewards/rejected": -0.005344724282622337,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": -3.3708367347717285,
      "debug/policy_chosen_logps": -134.6257781982422,
      "debug/policy_rejected_logits": -3.2685859203338623,
      "debug/policy_rejected_logps": -164.87184143066406,
      "debug/reference_chosen_logps": -134.9357452392578,
      "debug/reference_rejected_logps": -164.06057739257812,
      "epoch": 0.7027027027027027,
      "grad_norm": 3.5262056978807492,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3708367347717285,
      "logits/rejected": -3.2685859203338623,
      "logps/chosen": -134.6257781982422,
      "logps/rejected": -164.87184143066406,
      "loss": 0.4954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0030995942652225494,
      "rewards/margins": 0.011212329380214214,
      "rewards/rejected": -0.008112735114991665,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": -3.276146650314331,
      "debug/policy_chosen_logps": -127.90383911132812,
      "debug/policy_rejected_logits": -3.2199714183807373,
      "debug/policy_rejected_logps": -148.08279418945312,
      "debug/reference_chosen_logps": -127.79987335205078,
      "debug/reference_rejected_logps": -148.602783203125,
      "epoch": 0.7297297297297297,
      "grad_norm": 3.5419489962048973,
      "learning_rate": 1e-06,
      "logits/chosen": -3.276146650314331,
      "logits/rejected": -3.2199714183807373,
      "logps/chosen": -127.90383911132812,
      "logps/rejected": -148.08279418945312,
      "loss": 0.4972,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0010396192083135247,
      "rewards/margins": -0.006239423528313637,
      "rewards/rejected": 0.005199803970754147,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": -3.3671457767486572,
      "debug/policy_chosen_logps": -142.98072814941406,
      "debug/policy_rejected_logits": -3.313788652420044,
      "debug/policy_rejected_logps": -159.15054321289062,
      "debug/reference_chosen_logps": -143.54417419433594,
      "debug/reference_rejected_logps": -159.19894409179688,
      "epoch": 0.7567567567567568,
      "grad_norm": 3.6334386387375948,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3671457767486572,
      "logits/rejected": -3.313788652420044,
      "logps/chosen": -142.98072814941406,
      "logps/rejected": -159.15054321289062,
      "loss": 0.4957,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005634431727230549,
      "rewards/margins": 0.005150537472218275,
      "rewards/rejected": 0.0004838943714275956,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": -3.451542377471924,
      "debug/policy_chosen_logps": -144.62733459472656,
      "debug/policy_rejected_logits": -3.500192403793335,
      "debug/policy_rejected_logps": -129.41226196289062,
      "debug/reference_chosen_logps": -144.27435302734375,
      "debug/reference_rejected_logps": -128.7982177734375,
      "epoch": 0.7837837837837838,
      "grad_norm": 3.525191396719062,
      "learning_rate": 1e-06,
      "logits/chosen": -3.451542377471924,
      "logits/rejected": -3.500192403793335,
      "logps/chosen": -144.62733459472656,
      "logps/rejected": -129.41226196289062,
      "loss": 0.4925,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003529853653162718,
      "rewards/margins": 0.0026106643490493298,
      "rewards/rejected": -0.006140518002212048,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": -3.4419167041778564,
      "debug/policy_chosen_logps": -128.94613647460938,
      "debug/policy_rejected_logits": -3.3551814556121826,
      "debug/policy_rejected_logps": -125.87129974365234,
      "debug/reference_chosen_logps": -129.1351318359375,
      "debug/reference_rejected_logps": -125.61131286621094,
      "epoch": 0.8108108108108109,
      "grad_norm": 3.4357174715493812,
      "learning_rate": 1e-06,
      "logits/chosen": -3.4419167041778564,
      "logits/rejected": -3.3551814556121826,
      "logps/chosen": -128.94613647460938,
      "logps/rejected": -125.87129974365234,
      "loss": 0.4956,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.001890067826025188,
      "rewards/margins": 0.00448994617909193,
      "rewards/rejected": -0.0025998782366514206,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": -3.464578628540039,
      "debug/policy_chosen_logps": -148.08114624023438,
      "debug/policy_rejected_logits": -3.42940354347229,
      "debug/policy_rejected_logps": -155.03309631347656,
      "debug/reference_chosen_logps": -147.857666015625,
      "debug/reference_rejected_logps": -153.7999267578125,
      "epoch": 0.8378378378378378,
      "grad_norm": 3.569444014406672,
      "learning_rate": 1e-06,
      "logits/chosen": -3.464578628540039,
      "logits/rejected": -3.42940354347229,
      "logps/chosen": -148.08114624023438,
      "logps/rejected": -155.03309631347656,
      "loss": 0.4994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0022347732447087765,
      "rewards/margins": 0.010096902027726173,
      "rewards/rejected": -0.012331675738096237,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": -3.2954604625701904,
      "debug/policy_chosen_logps": -146.03802490234375,
      "debug/policy_rejected_logits": -3.384243965148926,
      "debug/policy_rejected_logps": -145.24456787109375,
      "debug/reference_chosen_logps": -146.03688049316406,
      "debug/reference_rejected_logps": -144.96478271484375,
      "epoch": 0.8648648648648649,
      "grad_norm": 3.794862233420595,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2954604625701904,
      "logits/rejected": -3.384243965148926,
      "logps/chosen": -146.03802490234375,
      "logps/rejected": -145.24456787109375,
      "loss": 0.4995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.1482159607112408e-05,
      "rewards/margins": 0.002786235883831978,
      "rewards/rejected": -0.0027977179270237684,
      "step": 32
    },
    {
      "debug/policy_chosen_logits": -3.393167734146118,
      "debug/policy_chosen_logps": -135.54547119140625,
      "debug/policy_rejected_logits": -3.4107766151428223,
      "debug/policy_rejected_logps": -126.21063995361328,
      "debug/reference_chosen_logps": -135.9659423828125,
      "debug/reference_rejected_logps": -123.26311492919922,
      "epoch": 0.8918918918918919,
      "grad_norm": 3.7044090203550195,
      "learning_rate": 1e-06,
      "logits/chosen": -3.393167734146118,
      "logits/rejected": -3.4107766151428223,
      "logps/chosen": -135.54547119140625,
      "logps/rejected": -126.21063995361328,
      "loss": 0.4985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004204845521599054,
      "rewards/margins": 0.03368005529046059,
      "rewards/rejected": -0.02947521209716797,
      "step": 33
    },
    {
      "debug/policy_chosen_logits": -3.3235180377960205,
      "debug/policy_chosen_logps": -159.286865234375,
      "debug/policy_rejected_logits": -3.369253635406494,
      "debug/policy_rejected_logps": -165.31944274902344,
      "debug/reference_chosen_logps": -158.15670776367188,
      "debug/reference_rejected_logps": -164.94976806640625,
      "epoch": 0.918918918918919,
      "grad_norm": 3.550860802017907,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3235180377960205,
      "logits/rejected": -3.369253635406494,
      "logps/chosen": -159.286865234375,
      "logps/rejected": -165.31944274902344,
      "loss": 0.4997,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.011301536113023758,
      "rewards/margins": -0.007604693062603474,
      "rewards/rejected": -0.0036968423519283533,
      "step": 34
    },
    {
      "debug/policy_chosen_logits": -3.367034673690796,
      "debug/policy_chosen_logps": -129.437744140625,
      "debug/policy_rejected_logits": -3.396902561187744,
      "debug/policy_rejected_logps": -157.3302001953125,
      "debug/reference_chosen_logps": -129.46514892578125,
      "debug/reference_rejected_logps": -157.58755493164062,
      "epoch": 0.9459459459459459,
      "grad_norm": 3.6785421821803044,
      "learning_rate": 1e-06,
      "logits/chosen": -3.367034673690796,
      "logits/rejected": -3.396902561187744,
      "logps/chosen": -129.437744140625,
      "logps/rejected": -157.3302001953125,
      "loss": 0.498,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00027409568428993225,
      "rewards/margins": -0.002299328101798892,
      "rewards/rejected": 0.002573424018919468,
      "step": 35
    },
    {
      "debug/policy_chosen_logits": -3.252328634262085,
      "debug/policy_chosen_logps": -139.81039428710938,
      "debug/policy_rejected_logits": -3.3797786235809326,
      "debug/policy_rejected_logps": -127.12653350830078,
      "debug/reference_chosen_logps": -140.35733032226562,
      "debug/reference_rejected_logps": -127.09129333496094,
      "epoch": 0.972972972972973,
      "grad_norm": 3.7364723193101446,
      "learning_rate": 1e-06,
      "logits/chosen": -3.252328634262085,
      "logits/rejected": -3.3797786235809326,
      "logps/chosen": -139.81039428710938,
      "logps/rejected": -127.12653350830078,
      "loss": 0.4951,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005469245836138725,
      "rewards/margins": 0.0058215707540512085,
      "rewards/rejected": -0.0003523253835737705,
      "step": 36
    },
    {
      "debug/policy_chosen_logits": -3.166999101638794,
      "debug/policy_chosen_logps": -165.9640350341797,
      "debug/policy_rejected_logits": -3.1885428428649902,
      "debug/policy_rejected_logps": -135.82620239257812,
      "debug/reference_chosen_logps": -164.97256469726562,
      "debug/reference_rejected_logps": -135.76358032226562,
      "epoch": 1.0,
      "grad_norm": 3.5142818885997116,
      "learning_rate": 1e-06,
      "logits/chosen": -3.166999101638794,
      "logits/rejected": -3.1885428428649902,
      "logps/chosen": -165.9640350341797,
      "logps/rejected": -135.82620239257812,
      "loss": 0.481,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009914855472743511,
      "rewards/margins": -0.009288596920669079,
      "rewards/rejected": -0.0006262586684897542,
      "step": 37
    },
    {
      "epoch": 1.0,
      "step": 37,
      "total_flos": 0.0,
      "train_loss": 0.4980396749200048,
      "train_runtime": 139.9378,
      "train_samples_per_second": 16.757,
      "train_steps_per_second": 0.264
    }
  ],
  "logging_steps": 1,
  "max_steps": 37,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}