{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1563,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 3.1847133757961784e-09,
      "logits/generated": -3.1013035774230957,
      "logits/real": -2.9071245193481445,
      "logps/generated": -110.83032989501953,
      "logps/real": -115.27798461914062,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/generated": 0.0,
      "rewards/margins": 0.0,
      "rewards/real": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.184713375796178e-08,
      "logits/generated": -3.0959415435791016,
      "logits/real": -2.8981242179870605,
      "logps/generated": -138.69400024414062,
      "logps/real": -131.35594177246094,
      "loss": 0.6776,
      "rewards/accuracies": 0.6944444179534912,
      "rewards/generated": -0.02448221482336521,
      "rewards/margins": 0.035542234778404236,
      "rewards/real": 0.011060023680329323,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.369426751592356e-08,
      "logits/generated": -3.099088430404663,
      "logits/real": -2.867945671081543,
      "logps/generated": -140.5944366455078,
      "logps/real": -127.7799301147461,
      "loss": 0.4267,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -0.5366789102554321,
      "rewards/margins": 0.748847484588623,
      "rewards/real": 0.21216857433319092,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.554140127388536e-08,
      "logits/generated": -3.0423760414123535,
      "logits/real": -2.886434316635132,
      "logps/generated": -136.33938598632812,
      "logps/real": -135.77671813964844,
      "loss": 0.1966,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.3359780311584473,
      "rewards/margins": 1.8330228328704834,
      "rewards/real": 0.4970448613166809,
      "step": 30
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2738853503184713e-07,
      "logits/generated": -3.0136947631835938,
      "logits/real": -2.7968850135803223,
      "logps/generated": -149.3777618408203,
      "logps/real": -126.96885681152344,
      "loss": 0.1114,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.9948108196258545,
      "rewards/margins": 2.8178465366363525,
      "rewards/real": 0.8230355978012085,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.592356687898089e-07,
      "logits/generated": -2.986863851547241,
      "logits/real": -2.745471715927124,
      "logps/generated": -158.6234588623047,
      "logps/real": -118.46070861816406,
      "loss": 0.0724,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.7238974571228027,
      "rewards/margins": 3.6493821144104004,
      "rewards/real": 0.9254843592643738,
      "step": 50
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9108280254777072e-07,
      "logits/generated": -2.930187702178955,
      "logits/real": -2.784818172454834,
      "logps/generated": -167.3131103515625,
      "logps/real": -126.73246765136719,
      "loss": 0.067,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.0826990604400635,
      "rewards/margins": 4.143509864807129,
      "rewards/real": 1.0608108043670654,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.2292993630573247e-07,
      "logits/generated": -2.9472126960754395,
      "logits/real": -2.7313473224639893,
      "logps/generated": -166.8314666748047,
      "logps/real": -132.07711791992188,
      "loss": 0.0529,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.4993481636047363,
      "rewards/margins": 4.573017597198486,
      "rewards/real": 1.0736693143844604,
      "step": 70
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5477707006369425e-07,
      "logits/generated": -2.943176746368408,
      "logits/real": -2.6685433387756348,
      "logps/generated": -169.8344268798828,
      "logps/real": -117.3721923828125,
      "loss": 0.043,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.954730272293091,
      "rewards/margins": 4.957295894622803,
      "rewards/real": 1.0025657415390015,
      "step": 80
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.86624203821656e-07,
      "logits/generated": -2.9318885803222656,
      "logits/real": -2.675168991088867,
      "logps/generated": -176.53231811523438,
      "logps/real": -127.081787109375,
      "loss": 0.0301,
      "rewards/accuracies": 1.0,
      "rewards/generated": -4.720462322235107,
      "rewards/margins": 5.908375263214111,
      "rewards/real": 1.1879123449325562,
      "step": 90
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.184713375796178e-07,
      "logits/generated": -2.906761884689331,
      "logits/real": -2.6541061401367188,
      "logps/generated": -189.25946044921875,
      "logps/real": -116.45599365234375,
      "loss": 0.0257,
      "rewards/accuracies": 1.0,
      "rewards/generated": -5.607656955718994,
      "rewards/margins": 6.690671443939209,
      "rewards/real": 1.083013892173767,
      "step": 100
    },
    {
      "epoch": 0.06,
      "eval_logits/generated": -2.884312152862549,
      "eval_logits/real": -2.651992082595825,
      "eval_logps/generated": -185.6055450439453,
      "eval_logps/real": -124.60718536376953,
      "eval_loss": 0.02876817248761654,
      "eval_rewards/accuracies": 0.9928343892097473,
      "eval_rewards/generated": -5.7769317626953125,
      "eval_rewards/margins": 6.782776355743408,
      "eval_rewards/real": 1.0058448314666748,
      "eval_runtime": 355.6798,
      "eval_samples_per_second": 14.058,
      "eval_steps_per_second": 0.441,
      "step": 100
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5031847133757957e-07,
      "logits/generated": -2.8794760704040527,
      "logits/real": -2.660065174102783,
      "logps/generated": -193.64273071289062,
      "logps/real": -135.76461791992188,
      "loss": 0.0255,
      "rewards/accuracies": 1.0,
      "rewards/generated": -6.355246543884277,
      "rewards/margins": 7.2249555587768555,
      "rewards/real": 0.869709312915802,
      "step": 110
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8216560509554143e-07,
      "logits/generated": -2.8529515266418457,
      "logits/real": -2.640996217727661,
      "logps/generated": -197.51763916015625,
      "logps/real": -123.4830322265625,
      "loss": 0.0231,
      "rewards/accuracies": 1.0,
      "rewards/generated": -6.6658616065979,
      "rewards/margins": 7.497047424316406,
      "rewards/real": 0.8311867713928223,
      "step": 120
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.140127388535032e-07,
      "logits/generated": -2.842136859893799,
      "logits/real": -2.6427853107452393,
      "logps/generated": -211.53878784179688,
      "logps/real": -134.64431762695312,
      "loss": 0.0202,
      "rewards/accuracies": 1.0,
      "rewards/generated": -7.736472129821777,
      "rewards/margins": 8.344034194946289,
      "rewards/real": 0.6075613498687744,
      "step": 130
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.4585987261146494e-07,
      "logits/generated": -2.7967801094055176,
      "logits/real": -2.5978636741638184,
      "logps/generated": -211.2340087890625,
      "logps/real": -119.82623291015625,
      "loss": 0.0118,
      "rewards/accuracies": 1.0,
      "rewards/generated": -8.243621826171875,
      "rewards/margins": 8.861018180847168,
      "rewards/real": 0.6173967123031616,
      "step": 140
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.777070063694267e-07,
      "logits/generated": -2.7344508171081543,
      "logits/real": -2.5488593578338623,
      "logps/generated": -224.51187133789062,
      "logps/real": -135.88230895996094,
      "loss": 0.0161,
      "rewards/accuracies": 1.0,
      "rewards/generated": -9.361867904663086,
      "rewards/margins": 9.38014030456543,
      "rewards/real": 0.01827339455485344,
      "step": 150
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.989331436699858e-07,
      "logits/generated": -2.591235399246216,
      "logits/real": -2.2508230209350586,
      "logps/generated": -239.1712188720703,
      "logps/real": -137.87863159179688,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/generated": -11.244747161865234,
      "rewards/margins": 10.78372859954834,
      "rewards/real": -0.46101751923561096,
      "step": 160
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.953769559032717e-07,
      "logits/generated": -2.6561527252197266,
      "logits/real": -2.3267903327941895,
      "logps/generated": -236.52676391601562,
      "logps/real": -128.25636291503906,
      "loss": 0.0127,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.687575340270996,
      "rewards/margins": 10.42524528503418,
      "rewards/real": -0.26232948899269104,
      "step": 170
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.918207681365576e-07,
      "logits/generated": -2.689512014389038,
      "logits/real": -2.4082565307617188,
      "logps/generated": -224.41653442382812,
      "logps/real": -143.75363159179688,
      "loss": 0.0083,
      "rewards/accuracies": 1.0,
      "rewards/generated": -9.677858352661133,
      "rewards/margins": 10.139410018920898,
      "rewards/real": 0.4615510106086731,
      "step": 180
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.882645803698435e-07,
      "logits/generated": -2.6560425758361816,
      "logits/real": -2.3908610343933105,
      "logps/generated": -238.35336303710938,
      "logps/real": -136.33726501464844,
      "loss": 0.0092,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.994186401367188,
      "rewards/margins": 11.490355491638184,
      "rewards/real": 0.49616795778274536,
      "step": 190
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.847083926031294e-07,
      "logits/generated": -2.649038791656494,
      "logits/real": -2.2306861877441406,
      "logps/generated": -275.4481201171875,
      "logps/real": -132.113037109375,
      "loss": 0.0096,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.323870658874512,
      "rewards/margins": 12.914576530456543,
      "rewards/real": -0.4092935025691986,
      "step": 200
    },
    {
      "epoch": 0.13,
      "eval_logits/generated": -2.594467878341675,
      "eval_logits/real": -2.2412638664245605,
      "eval_logps/generated": -254.09414672851562,
      "eval_logps/real": -136.21932983398438,
      "eval_loss": 0.012603986077010632,
      "eval_rewards/accuracies": 0.9984076619148254,
      "eval_rewards/generated": -12.62579345703125,
      "eval_rewards/margins": 12.470422744750977,
      "eval_rewards/real": -0.1553698629140854,
      "eval_runtime": 354.1472,
      "eval_samples_per_second": 14.118,
      "eval_steps_per_second": 0.443,
      "step": 200
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.811522048364154e-07,
      "logits/generated": -2.56303334236145,
      "logits/real": -2.194331169128418,
      "logps/generated": -241.0314178466797,
      "logps/real": -127.5682373046875,
      "loss": 0.0299,
      "rewards/accuracies": 1.0,
      "rewards/generated": -12.397547721862793,
      "rewards/margins": 11.594755172729492,
      "rewards/real": -0.802793025970459,
      "step": 210
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.775960170697012e-07,
      "logits/generated": -2.7050158977508545,
      "logits/real": -2.284231424331665,
      "logps/generated": -245.81307983398438,
      "logps/real": -135.11415100097656,
      "loss": 0.0145,
      "rewards/accuracies": 1.0,
      "rewards/generated": -11.418529510498047,
      "rewards/margins": 9.91020393371582,
      "rewards/real": -1.5083262920379639,
      "step": 220
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7403982930298717e-07,
      "logits/generated": -2.615196704864502,
      "logits/real": -2.214932918548584,
      "logps/generated": -277.31927490234375,
      "logps/real": -153.6757049560547,
      "loss": 0.0137,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.367179870605469,
      "rewards/margins": 11.812820434570312,
      "rewards/real": -2.5543580055236816,
      "step": 230
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7048364153627306e-07,
      "logits/generated": -2.5809149742126465,
      "logits/real": -2.3492672443389893,
      "logps/generated": -271.6045227050781,
      "logps/real": -177.99237060546875,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.206598281860352,
      "rewards/margins": 12.018148422241211,
      "rewards/real": -2.188450336456299,
      "step": 240
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.66927453769559e-07,
      "logits/generated": -2.5388941764831543,
      "logits/real": -2.2705929279327393,
      "logps/generated": -270.50408935546875,
      "logps/real": -148.82215881347656,
      "loss": 0.0168,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.393671989440918,
      "rewards/margins": 11.958968162536621,
      "rewards/real": -2.4347033500671387,
      "step": 250
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.633712660028449e-07,
      "logits/generated": -2.637037754058838,
      "logits/real": -2.2578437328338623,
      "logps/generated": -250.59121704101562,
      "logps/real": -128.23684692382812,
      "loss": 0.014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -11.980981826782227,
      "rewards/margins": 11.673439025878906,
      "rewards/real": -0.30754321813583374,
      "step": 260
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.5981507823613085e-07,
      "logits/generated": -2.7627644538879395,
      "logits/real": -2.487227201461792,
      "logps/generated": -216.2014617919922,
      "logps/real": -121.24652099609375,
      "loss": 0.0131,
      "rewards/accuracies": 1.0,
      "rewards/generated": -8.369101524353027,
      "rewards/margins": 9.171772956848145,
      "rewards/real": 0.8026714324951172,
      "step": 270
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.562588904694168e-07,
      "logits/generated": -2.71317720413208,
      "logits/real": -2.4743077754974365,
      "logps/generated": -222.00350952148438,
      "logps/real": -134.9727325439453,
      "loss": 0.0099,
      "rewards/accuracies": 1.0,
      "rewards/generated": -9.607930183410645,
      "rewards/margins": 9.943489074707031,
      "rewards/real": 0.33555930852890015,
      "step": 280
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.5270270270270264e-07,
      "logits/generated": -2.706022024154663,
      "logits/real": -2.4252126216888428,
      "logps/generated": -249.82839965820312,
      "logps/real": -131.52957153320312,
      "loss": 0.0107,
      "rewards/accuracies": 1.0,
      "rewards/generated": -11.630704879760742,
      "rewards/margins": 11.901201248168945,
      "rewards/real": 0.270496666431427,
      "step": 290
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.491465149359886e-07,
      "logits/generated": -2.7073264122009277,
      "logits/real": -2.37262225151062,
      "logps/generated": -234.38796997070312,
      "logps/real": -122.046630859375,
      "loss": 0.024,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.743330001831055,
      "rewards/margins": 11.167850494384766,
      "rewards/real": 0.4245213568210602,
      "step": 300
    },
    {
      "epoch": 0.19,
      "eval_logits/generated": -2.7227139472961426,
      "eval_logits/real": -2.5040037631988525,
      "eval_logps/generated": -238.78199768066406,
      "eval_logps/real": -133.49249267578125,
      "eval_loss": 0.012583808973431587,
      "eval_rewards/accuracies": 0.9968152642250061,
      "eval_rewards/generated": -11.094578742980957,
      "eval_rewards/margins": 11.211891174316406,
      "eval_rewards/real": 0.11731348186731339,
      "eval_runtime": 358.5504,
      "eval_samples_per_second": 13.945,
      "eval_steps_per_second": 0.438,
      "step": 300
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.4559032716927454e-07,
      "logits/generated": -2.7238636016845703,
      "logits/real": -2.5220375061035156,
      "logps/generated": -257.8681640625,
      "logps/real": -148.02816772460938,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/generated": -11.876145362854004,
      "rewards/margins": 11.944305419921875,
      "rewards/real": 0.06815892457962036,
      "step": 310
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.420341394025605e-07,
      "logits/generated": -2.6920430660247803,
      "logits/real": -2.4902937412261963,
      "logps/generated": -246.45480346679688,
      "logps/real": -140.23394775390625,
      "loss": 0.0118,
      "rewards/accuracies": 1.0,
      "rewards/generated": -11.85106372833252,
      "rewards/margins": 11.594002723693848,
      "rewards/real": -0.2570618987083435,
      "step": 320
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.384779516358463e-07,
      "logits/generated": -2.7012107372283936,
      "logits/real": -2.5255706310272217,
      "logps/generated": -236.407470703125,
      "logps/real": -141.53895568847656,
      "loss": 0.014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.566434860229492,
      "rewards/margins": 10.503705024719238,
      "rewards/real": -0.06273023784160614,
      "step": 330
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.3492176386913227e-07,
      "logits/generated": -2.703942060470581,
      "logits/real": -2.521278142929077,
      "logps/generated": -258.814208984375,
      "logps/real": -150.25686645507812,
      "loss": 0.0056,
      "rewards/accuracies": 1.0,
      "rewards/generated": -12.056513786315918,
      "rewards/margins": 11.703389167785645,
      "rewards/real": -0.3531256914138794,
      "step": 340
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.313655761024182e-07,
      "logits/generated": -2.696030378341675,
      "logits/real": -2.461880922317505,
      "logps/generated": -268.0128173828125,
      "logps/real": -144.72943115234375,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.055676460266113,
      "rewards/margins": 12.351667404174805,
      "rewards/real": -0.704009473323822,
      "step": 350
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.278093883357041e-07,
      "logits/generated": -2.7000229358673096,
      "logits/real": -2.4609451293945312,
      "logps/generated": -257.57745361328125,
      "logps/real": -132.90365600585938,
      "loss": 0.0151,
      "rewards/accuracies": 1.0,
      "rewards/generated": -12.664046287536621,
      "rewards/margins": 12.791855812072754,
      "rewards/real": 0.12780967354774475,
      "step": 360
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.2425320056899e-07,
      "logits/generated": -2.660365343093872,
      "logits/real": -2.428536891937256,
      "logps/generated": -251.99588012695312,
      "logps/real": -153.57388305664062,
      "loss": 0.0041,
      "rewards/accuracies": 1.0,
      "rewards/generated": -12.598872184753418,
      "rewards/margins": 12.66786003112793,
      "rewards/real": 0.06898676604032516,
      "step": 370
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.2069701280227595e-07,
      "logits/generated": -2.6655094623565674,
      "logits/real": -2.4059395790100098,
      "logps/generated": -268.7626953125,
      "logps/real": -157.2294464111328,
      "loss": 0.0148,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -13.887835502624512,
      "rewards/margins": 12.452804565429688,
      "rewards/real": -1.4350312948226929,
      "step": 380
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.1714082503556185e-07,
      "logits/generated": -2.717266798019409,
      "logits/real": -2.5067367553710938,
      "logps/generated": -264.3714599609375,
      "logps/real": -148.14230346679688,
      "loss": 0.0111,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.594515800476074,
      "rewards/margins": 12.036942481994629,
      "rewards/real": -1.5575740337371826,
      "step": 390
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.135846372688478e-07,
      "logits/generated": -2.7214503288269043,
      "logits/real": -2.450742244720459,
      "logps/generated": -262.1417541503906,
      "logps/real": -143.6311798095703,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/generated": -13.615748405456543,
      "rewards/margins": 13.03248119354248,
      "rewards/real": -0.5832666754722595,
      "step": 400
    },
    {
      "epoch": 0.26,
      "eval_logits/generated": -2.702763080596924,
      "eval_logits/real": -2.4737653732299805,
      "eval_logps/generated": -264.14105224609375,
      "eval_logps/real": -136.62977600097656,
      "eval_loss": 0.008167657069861889,
      "eval_rewards/accuracies": 0.9984076619148254,
      "eval_rewards/generated": -13.630484580993652,
      "eval_rewards/margins": 13.434069633483887,
      "eval_rewards/real": -0.19641424715518951,
      "eval_runtime": 353.3453,
      "eval_samples_per_second": 14.15,
      "eval_steps_per_second": 0.444,
      "step": 400
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.100284495021337e-07,
      "logits/generated": -2.676800012588501,
      "logits/real": -2.4450738430023193,
      "logps/generated": -282.40460205078125,
      "logps/real": -134.112060546875,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.695889472961426,
      "rewards/margins": 14.696159362792969,
      "rewards/real": 0.0002695709408726543,
      "step": 410
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.064722617354196e-07,
      "logits/generated": -2.7011332511901855,
      "logits/real": -2.433877468109131,
      "logps/generated": -285.080078125,
      "logps/real": -144.6403350830078,
      "loss": 0.0042,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.842653274536133,
      "rewards/margins": 14.158676147460938,
      "rewards/real": -0.6839768290519714,
      "step": 420
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.0291607396870553e-07,
      "logits/generated": -2.6828908920288086,
      "logits/real": -2.396275043487549,
      "logps/generated": -274.49346923828125,
      "logps/real": -135.5173797607422,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.000213623046875,
      "rewards/margins": 14.3043851852417,
      "rewards/real": -0.6958280801773071,
      "step": 430
    },
    {
      "epoch": 0.28,
      "learning_rate": 3.993598862019915e-07,
      "logits/generated": -2.6642165184020996,
      "logits/real": -2.3830320835113525,
      "logps/generated": -289.18939208984375,
      "logps/real": -149.1437530517578,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.230016708374023,
      "rewards/margins": 14.582804679870605,
      "rewards/real": -1.6472117900848389,
      "step": 440
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.9580369843527737e-07,
      "logits/generated": -2.66624116897583,
      "logits/real": -2.3914172649383545,
      "logps/generated": -312.4716796875,
      "logps/real": -150.9366912841797,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.956409454345703,
      "rewards/margins": 15.78361701965332,
      "rewards/real": -2.1727941036224365,
      "step": 450
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.9224751066856327e-07,
      "logits/generated": -2.5691237449645996,
      "logits/real": -2.455709934234619,
      "logps/generated": -310.66326904296875,
      "logps/real": -183.9186553955078,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.07615089416504,
      "rewards/margins": 15.606036186218262,
      "rewards/real": -2.4701130390167236,
      "step": 460
    },
    {
      "epoch": 0.3,
      "learning_rate": 3.886913229018492e-07,
      "logits/generated": -2.6210334300994873,
      "logits/real": -2.333034038543701,
      "logps/generated": -283.49578857421875,
      "logps/real": -139.85386657714844,
      "loss": 0.0078,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.575042724609375,
      "rewards/margins": 15.00732707977295,
      "rewards/real": -0.5677127242088318,
      "step": 470
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.851351351351351e-07,
      "logits/generated": -2.5755233764648438,
      "logits/real": -2.3314523696899414,
      "logps/generated": -289.11907958984375,
      "logps/real": -146.9121551513672,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.91128921508789,
      "rewards/margins": 16.000783920288086,
      "rewards/real": -0.9105021357536316,
      "step": 480
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.8157894736842105e-07,
      "logits/generated": -2.615877628326416,
      "logits/real": -2.3037173748016357,
      "logps/generated": -293.7407531738281,
      "logps/real": -138.11875915527344,
      "loss": 0.0099,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.228160858154297,
      "rewards/margins": 14.908615112304688,
      "rewards/real": -1.319542646408081,
      "step": 490
    },
    {
      "epoch": 0.32,
      "learning_rate": 3.7802275960170695e-07,
      "logits/generated": -2.6651482582092285,
      "logits/real": -2.4918980598449707,
      "logps/generated": -273.33746337890625,
      "logps/real": -137.34373474121094,
      "loss": 0.0073,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.052767753601074,
      "rewards/margins": 13.785321235656738,
      "rewards/real": -0.26744553446769714,
      "step": 500
    },
    {
      "epoch": 0.32,
      "eval_logits/generated": -2.6477458477020264,
      "eval_logits/real": -2.4284629821777344,
      "eval_logps/generated": -262.2040100097656,
      "eval_logps/real": -133.81561279296875,
      "eval_loss": 0.00813828781247139,
      "eval_rewards/accuracies": 0.9984076619148254,
      "eval_rewards/generated": -13.436781883239746,
      "eval_rewards/margins": 13.521784782409668,
      "eval_rewards/real": 0.08500289916992188,
      "eval_runtime": 358.267,
      "eval_samples_per_second": 13.956,
      "eval_steps_per_second": 0.438,
      "step": 500
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.7446657183499284e-07,
      "logits/generated": -2.6251749992370605,
      "logits/real": -2.447301149368286,
      "logps/generated": -277.0758972167969,
      "logps/real": -132.02191162109375,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.138468742370605,
      "rewards/margins": 14.193331718444824,
      "rewards/real": 0.05486304685473442,
      "step": 510
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.709103840682788e-07,
      "logits/generated": -2.637619972229004,
      "logits/real": -2.458965301513672,
      "logps/generated": -268.1296081542969,
      "logps/real": -129.8132781982422,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.191378593444824,
      "rewards/margins": 14.115701675415039,
      "rewards/real": -0.07567773759365082,
      "step": 520
    },
    {
      "epoch": 0.34,
      "learning_rate": 3.6735419630156474e-07,
      "logits/generated": -2.642280101776123,
      "logits/real": -2.3937056064605713,
      "logps/generated": -273.968994140625,
      "logps/real": -132.57595825195312,
      "loss": 0.0076,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -14.767156600952148,
      "rewards/margins": 13.815185546875,
      "rewards/real": -0.9519737958908081,
      "step": 530
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.637980085348506e-07,
      "logits/generated": -2.6142544746398926,
      "logits/real": -2.3965210914611816,
      "logps/generated": -291.0428161621094,
      "logps/real": -143.89450073242188,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.896871566772461,
      "rewards/margins": 14.926864624023438,
      "rewards/real": -0.9700061678886414,
      "step": 540
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.602418207681365e-07,
      "logits/generated": -2.547969102859497,
      "logits/real": -2.314603805541992,
      "logps/generated": -292.73193359375,
      "logps/real": -152.8965301513672,
      "loss": 0.0038,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.91225242614746,
      "rewards/margins": 15.625356674194336,
      "rewards/real": -1.286894679069519,
      "step": 550
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.5668563300142247e-07,
      "logits/generated": -2.5611112117767334,
      "logits/real": -2.341109037399292,
      "logps/generated": -326.85406494140625,
      "logps/real": -146.52040100097656,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.382369995117188,
      "rewards/margins": 17.716310501098633,
      "rewards/real": -1.6660608053207397,
      "step": 560
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.5312944523470837e-07,
      "logits/generated": -2.522569417953491,
      "logits/real": -2.2818217277526855,
      "logps/generated": -307.17657470703125,
      "logps/real": -154.59783935546875,
      "loss": 0.0105,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -18.03821563720703,
      "rewards/margins": 15.825765609741211,
      "rewards/real": -2.2124505043029785,
      "step": 570
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.495732574679943e-07,
      "logits/generated": -2.546279191970825,
      "logits/real": -2.276088237762451,
      "logps/generated": -326.0049743652344,
      "logps/real": -164.81809997558594,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.003278732299805,
      "rewards/margins": 16.616857528686523,
      "rewards/real": -3.3864219188690186,
      "step": 580
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.460170697012802e-07,
      "logits/generated": -2.621000051498413,
      "logits/real": -2.389765977859497,
      "logps/generated": -319.06060791015625,
      "logps/real": -164.0532684326172,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.283437728881836,
      "rewards/margins": 15.354583740234375,
      "rewards/real": -2.928854465484619,
      "step": 590
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.424608819345661e-07,
      "logits/generated": -2.6297097206115723,
      "logits/real": -2.3201308250427246,
      "logps/generated": -322.4084167480469,
      "logps/real": -152.594482421875,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.64119529724121,
      "rewards/margins": 16.223892211914062,
      "rewards/real": -2.417302370071411,
      "step": 600
    },
    {
      "epoch": 0.38,
      "eval_logits/generated": -2.595550060272217,
      "eval_logits/real": -2.381129264831543,
      "eval_logps/generated": -312.4772033691406,
      "eval_logps/real": -163.40432739257812,
      "eval_loss": 0.007091078907251358,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -18.46409797668457,
      "eval_rewards/margins": 15.590229034423828,
      "eval_rewards/real": -2.873868703842163,
      "eval_runtime": 353.487,
      "eval_samples_per_second": 14.145,
      "eval_steps_per_second": 0.444,
      "step": 600
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.3890469416785205e-07,
      "logits/generated": -2.6035828590393066,
      "logits/real": -2.3773751258850098,
      "logps/generated": -321.2727966308594,
      "logps/real": -163.73019409179688,
      "loss": 0.0087,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.917009353637695,
      "rewards/margins": 15.915303230285645,
      "rewards/real": -3.0017073154449463,
      "step": 610
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.35348506401138e-07,
      "logits/generated": -2.6622188091278076,
      "logits/real": -2.389137029647827,
      "logps/generated": -283.23095703125,
      "logps/real": -136.14297485351562,
      "loss": 0.0053,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.39533805847168,
      "rewards/margins": 14.832684516906738,
      "rewards/real": -0.5626530051231384,
      "step": 620
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.3179231863442384e-07,
      "logits/generated": -2.605447292327881,
      "logits/real": -2.3979125022888184,
      "logps/generated": -284.0592041015625,
      "logps/real": -152.1702880859375,
      "loss": 0.0052,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.38554573059082,
      "rewards/margins": 14.56556224822998,
      "rewards/real": -0.8199828863143921,
      "step": 630
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.282361308677098e-07,
      "logits/generated": -2.576028347015381,
      "logits/real": -2.3189806938171387,
      "logps/generated": -303.54766845703125,
      "logps/real": -147.36514282226562,
      "loss": 0.0039,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.905027389526367,
      "rewards/margins": 15.85954475402832,
      "rewards/real": -1.045483112335205,
      "step": 640
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.2467994310099573e-07,
      "logits/generated": -2.55846905708313,
      "logits/real": -2.2951676845550537,
      "logps/generated": -308.7777404785156,
      "logps/real": -159.1045379638672,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.19515609741211,
      "rewards/margins": 16.93841552734375,
      "rewards/real": -1.2567408084869385,
      "step": 650
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.211237553342817e-07,
      "logits/generated": -2.5596654415130615,
      "logits/real": -2.2995269298553467,
      "logps/generated": -308.10760498046875,
      "logps/real": -145.70848083496094,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.918209075927734,
      "rewards/margins": 16.569629669189453,
      "rewards/real": -1.3485772609710693,
      "step": 660
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.175675675675675e-07,
      "logits/generated": -2.5624568462371826,
      "logits/real": -2.2460904121398926,
      "logps/generated": -308.0290832519531,
      "logps/real": -130.75479125976562,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.573022842407227,
      "rewards/margins": 16.77570152282715,
      "rewards/real": -0.797319769859314,
      "step": 670
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.1401137980085347e-07,
      "logits/generated": -2.581447124481201,
      "logits/real": -2.313119411468506,
      "logps/generated": -366.02130126953125,
      "logps/real": -138.63331604003906,
      "loss": 0.0066,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -22.266155242919922,
      "rewards/margins": 21.87325668334961,
      "rewards/real": -0.3929004669189453,
      "step": 680
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.104551920341394e-07,
      "logits/generated": -2.5475454330444336,
      "logits/real": -2.2901225090026855,
      "logps/generated": -328.03204345703125,
      "logps/real": -142.40859985351562,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.810443878173828,
      "rewards/margins": 18.20998191833496,
      "rewards/real": -0.6004606485366821,
      "step": 690
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.068990042674253e-07,
      "logits/generated": -2.5037403106689453,
      "logits/real": -2.134835720062256,
      "logps/generated": -323.8102722167969,
      "logps/real": -141.37823486328125,
      "loss": 0.0097,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.94583511352539,
      "rewards/margins": 18.077741622924805,
      "rewards/real": -1.868093490600586,
      "step": 700
    },
    {
      "epoch": 0.45,
      "eval_logits/generated": -2.5209903717041016,
      "eval_logits/real": -2.2045412063598633,
      "eval_logps/generated": -297.73382568359375,
      "eval_logps/real": -157.5738525390625,
      "eval_loss": 0.007741523906588554,
      "eval_rewards/accuracies": 0.9984076619148254,
      "eval_rewards/generated": -16.989761352539062,
      "eval_rewards/margins": 14.698938369750977,
      "eval_rewards/real": -2.290821075439453,
      "eval_runtime": 359.389,
      "eval_samples_per_second": 13.913,
      "eval_steps_per_second": 0.437,
      "step": 700
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.033428165007112e-07,
      "logits/generated": -2.552351236343384,
      "logits/real": -2.212099552154541,
      "logps/generated": -309.7628479003906,
      "logps/real": -141.3500518798828,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.07712173461914,
      "rewards/margins": 16.641395568847656,
      "rewards/real": -1.4357249736785889,
      "step": 710
    },
    {
      "epoch": 0.46,
      "learning_rate": 2.9978662873399715e-07,
      "logits/generated": -2.5791096687316895,
      "logits/real": -2.263129234313965,
      "logps/generated": -321.2320251464844,
      "logps/real": -147.78073120117188,
      "loss": 0.0025,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.866134643554688,
      "rewards/margins": 17.729503631591797,
      "rewards/real": -1.1366310119628906,
      "step": 720
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.9623044096728305e-07,
      "logits/generated": -2.6658735275268555,
      "logits/real": -2.530928611755371,
      "logps/generated": -284.85797119140625,
      "logps/real": -157.20460510253906,
      "loss": 0.0136,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -16.189687728881836,
      "rewards/margins": 15.173242568969727,
      "rewards/real": -1.016442894935608,
      "step": 730
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.92674253200569e-07,
      "logits/generated": -2.7502713203430176,
      "logits/real": -2.5832438468933105,
      "logps/generated": -276.59503173828125,
      "logps/real": -141.8132781982422,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/generated": -14.831079483032227,
      "rewards/margins": 14.12823486328125,
      "rewards/real": -0.7028436064720154,
      "step": 740
    },
    {
      "epoch": 0.48,
      "learning_rate": 2.8911806543385494e-07,
      "logits/generated": -2.728147506713867,
      "logits/real": -2.558288097381592,
      "logps/generated": -308.59332275390625,
      "logps/real": -154.87002563476562,
      "loss": 0.0126,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.347309112548828,
      "rewards/margins": 16.73641586303711,
      "rewards/real": -0.6108967661857605,
      "step": 750
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.855618776671408e-07,
      "logits/generated": -2.744182586669922,
      "logits/real": -2.6283411979675293,
      "logps/generated": -332.50518798828125,
      "logps/real": -180.13763427734375,
      "loss": 0.0056,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.751907348632812,
      "rewards/margins": 17.28380012512207,
      "rewards/real": -2.4681074619293213,
      "step": 760
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.8200568990042673e-07,
      "logits/generated": -2.758481979370117,
      "logits/real": -2.552919626235962,
      "logps/generated": -315.9869689941406,
      "logps/real": -153.39651489257812,
      "loss": 0.007,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.343120574951172,
      "rewards/margins": 16.596370697021484,
      "rewards/real": -1.7467491626739502,
      "step": 770
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.784495021337127e-07,
      "logits/generated": -2.720580816268921,
      "logits/real": -2.4772982597351074,
      "logps/generated": -295.3759460449219,
      "logps/real": -141.34848022460938,
      "loss": 0.008,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.396093368530273,
      "rewards/margins": 15.803703308105469,
      "rewards/real": -1.5923895835876465,
      "step": 780
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7489331436699857e-07,
      "logits/generated": -2.7093377113342285,
      "logits/real": -2.5702013969421387,
      "logps/generated": -339.5489807128906,
      "logps/real": -177.2974853515625,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.449642181396484,
      "rewards/margins": 18.11000633239746,
      "rewards/real": -3.339632034301758,
      "step": 790
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7133712660028446e-07,
      "logits/generated": -2.722670555114746,
      "logits/real": -2.518655300140381,
      "logps/generated": -336.6543884277344,
      "logps/real": -144.3495635986328,
      "loss": 0.0052,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -20.40723419189453,
      "rewards/margins": 18.609041213989258,
      "rewards/real": -1.7981945276260376,
      "step": 800
    },
    {
      "epoch": 0.51,
      "eval_logits/generated": -2.71830677986145,
      "eval_logits/real": -2.5409433841705322,
      "eval_logps/generated": -326.1592712402344,
      "eval_logps/real": -151.64842224121094,
      "eval_loss": 0.006455760914832354,
      "eval_rewards/accuracies": 0.9992038011550903,
      "eval_rewards/generated": -19.832304000854492,
      "eval_rewards/margins": 18.1340274810791,
      "eval_rewards/real": -1.698278784751892,
      "eval_runtime": 352.7904,
      "eval_samples_per_second": 14.173,
      "eval_steps_per_second": 0.445,
      "step": 800
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.677809388335704e-07,
      "logits/generated": -2.73547625541687,
      "logits/real": -2.518897294998169,
      "logps/generated": -324.478271484375,
      "logps/real": -146.60842895507812,
      "loss": 0.0025,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.922399520874023,
      "rewards/margins": 17.002269744873047,
      "rewards/real": -1.9201271533966064,
      "step": 810
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.642247510668563e-07,
      "logits/generated": -2.7113728523254395,
      "logits/real": -2.50697922706604,
      "logps/generated": -313.7933349609375,
      "logps/real": -149.09490966796875,
      "loss": 0.0155,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.239910125732422,
      "rewards/margins": 16.581689834594727,
      "rewards/real": -1.6582221984863281,
      "step": 820
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6066856330014225e-07,
      "logits/generated": -2.7217416763305664,
      "logits/real": -2.5234363079071045,
      "logps/generated": -287.1163024902344,
      "logps/real": -139.886962890625,
      "loss": 0.0066,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.454859733581543,
      "rewards/margins": 15.261648178100586,
      "rewards/real": -0.1932099163532257,
      "step": 830
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5711237553342815e-07,
      "logits/generated": -2.7132928371429443,
      "logits/real": -2.5606608390808105,
      "logps/generated": -294.3720397949219,
      "logps/real": -152.9593963623047,
      "loss": 0.0074,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.173110961914062,
      "rewards/margins": 15.697965621948242,
      "rewards/real": -0.47514596581459045,
      "step": 840
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5355618776671404e-07,
      "logits/generated": -2.6806228160858154,
      "logits/real": -2.4857285022735596,
      "logps/generated": -279.65948486328125,
      "logps/real": -146.00035095214844,
      "loss": 0.0075,
      "rewards/accuracies": 1.0,
      "rewards/generated": -15.32105541229248,
      "rewards/margins": 14.74870777130127,
      "rewards/real": -0.5723468065261841,
      "step": 850
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5e-07,
      "logits/generated": -2.6649889945983887,
      "logits/real": -2.503398895263672,
      "logps/generated": -286.626708984375,
      "logps/real": -137.37606811523438,
      "loss": 0.0055,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.276308059692383,
      "rewards/margins": 15.699457168579102,
      "rewards/real": -0.5768507122993469,
      "step": 860
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4644381223328594e-07,
      "logits/generated": -2.665027618408203,
      "logits/real": -2.5171055793762207,
      "logps/generated": -294.4344177246094,
      "logps/real": -142.8474884033203,
      "loss": 0.0025,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.496715545654297,
      "rewards/margins": 16.218063354492188,
      "rewards/real": -0.2786518335342407,
      "step": 870
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4288762446657183e-07,
      "logits/generated": -2.6565191745758057,
      "logits/real": -2.5109829902648926,
      "logps/generated": -332.05914306640625,
      "logps/real": -152.0934600830078,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.833471298217773,
      "rewards/margins": 18.865116119384766,
      "rewards/real": -0.9683563113212585,
      "step": 880
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.393314366998578e-07,
      "logits/generated": -2.697728395462036,
      "logits/real": -2.5354275703430176,
      "logps/generated": -308.3212890625,
      "logps/real": -153.72103881835938,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.799835205078125,
      "rewards/margins": 16.078920364379883,
      "rewards/real": -1.7209144830703735,
      "step": 890
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.3577524893314365e-07,
      "logits/generated": -2.679509162902832,
      "logits/real": -2.495677947998047,
      "logps/generated": -330.8416748046875,
      "logps/real": -143.41543579101562,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.74724769592285,
      "rewards/margins": 17.45358657836914,
      "rewards/real": -1.2936601638793945,
      "step": 900
    },
    {
      "epoch": 0.58,
      "eval_logits/generated": -2.6881191730499268,
      "eval_logits/real": -2.5333993434906006,
      "eval_logps/generated": -294.42584228515625,
      "eval_logps/real": -147.4919891357422,
      "eval_loss": 0.006735246162861586,
      "eval_rewards/accuracies": 0.9984076619148254,
      "eval_rewards/generated": -16.658964157104492,
      "eval_rewards/margins": 15.376328468322754,
      "eval_rewards/real": -1.282636284828186,
      "eval_runtime": 358.48,
      "eval_samples_per_second": 13.948,
      "eval_steps_per_second": 0.438,
      "step": 900
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.322190611664296e-07,
      "logits/generated": -2.6863019466400146,
      "logits/real": -2.584163188934326,
      "logps/generated": -306.78399658203125,
      "logps/real": -154.82162475585938,
      "loss": 0.0055,
      "rewards/accuracies": 1.0,
      "rewards/generated": -16.801063537597656,
      "rewards/margins": 16.25944709777832,
      "rewards/real": -0.5416165590286255,
      "step": 910
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.2866287339971549e-07,
      "logits/generated": -2.6918511390686035,
      "logits/real": -2.498774528503418,
      "logps/generated": -304.48968505859375,
      "logps/real": -154.81939697265625,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.309907913208008,
      "rewards/margins": 15.712469100952148,
      "rewards/real": -1.5974372625350952,
      "step": 920
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.251066856330014e-07,
      "logits/generated": -2.6575112342834473,
      "logits/real": -2.5105912685394287,
      "logps/generated": -300.7004089355469,
      "logps/real": -150.9338836669922,
      "loss": 0.0051,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.501060485839844,
      "rewards/margins": 14.903904914855957,
      "rewards/real": -2.597156047821045,
      "step": 930
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.2155049786628733e-07,
      "logits/generated": -2.672095537185669,
      "logits/real": -2.5284764766693115,
      "logps/generated": -307.1285705566406,
      "logps/real": -160.99423217773438,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.466632843017578,
      "rewards/margins": 15.776178359985352,
      "rewards/real": -1.6904557943344116,
      "step": 940
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1799431009957325e-07,
      "logits/generated": -2.66410756111145,
      "logits/real": -2.4963908195495605,
      "logps/generated": -316.0965881347656,
      "logps/real": -141.8169403076172,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.660221099853516,
      "rewards/margins": 16.873653411865234,
      "rewards/real": -1.7865674495697021,
      "step": 950
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1443812233285914e-07,
      "logits/generated": -2.685979127883911,
      "logits/real": -2.5782992839813232,
      "logps/generated": -327.9977722167969,
      "logps/real": -162.27938842773438,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.976680755615234,
      "rewards/margins": 17.09455108642578,
      "rewards/real": -1.8821306228637695,
      "step": 960
    },
    {
      "epoch": 0.62,
      "learning_rate": 2.108819345661451e-07,
      "logits/generated": -2.695758104324341,
      "logits/real": -2.485863208770752,
      "logps/generated": -313.2763671875,
      "logps/real": -153.49114990234375,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.86570167541504,
      "rewards/margins": 16.97518539428711,
      "rewards/real": -1.8905149698257446,
      "step": 970
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0732574679943098e-07,
      "logits/generated": -2.6816251277923584,
      "logits/real": -2.4063215255737305,
      "logps/generated": -310.80975341796875,
      "logps/real": -138.0583953857422,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.112398147583008,
      "rewards/margins": 16.26905632019043,
      "rewards/real": -1.8433430194854736,
      "step": 980
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0376955903271693e-07,
      "logits/generated": -2.676630973815918,
      "logits/real": -2.545987606048584,
      "logps/generated": -324.10198974609375,
      "logps/real": -170.13424682617188,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.194168090820312,
      "rewards/margins": 17.191312789916992,
      "rewards/real": -2.0028531551361084,
      "step": 990
    },
    {
      "epoch": 0.64,
      "learning_rate": 2.0021337126600283e-07,
      "logits/generated": -2.6588072776794434,
      "logits/real": -2.4335250854492188,
      "logps/generated": -307.8160705566406,
      "logps/real": -143.21780395507812,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.040515899658203,
      "rewards/margins": 16.0909423828125,
      "rewards/real": -1.949573278427124,
      "step": 1000
    },
    {
      "epoch": 0.64,
      "eval_logits/generated": -2.6403703689575195,
      "eval_logits/real": -2.469404697418213,
      "eval_logps/generated": -320.0989685058594,
      "eval_logps/real": -154.08863830566406,
      "eval_loss": 0.004710016772150993,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -19.226276397705078,
      "eval_rewards/margins": 17.283977508544922,
      "eval_rewards/real": -1.9422993659973145,
      "eval_runtime": 353.54,
      "eval_samples_per_second": 14.143,
      "eval_steps_per_second": 0.444,
      "step": 1000
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.9665718349928875e-07,
      "logits/generated": -2.6474626064300537,
      "logits/real": -2.3872134685516357,
      "logps/generated": -319.66595458984375,
      "logps/real": -144.20675659179688,
      "loss": 0.0091,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -19.061620712280273,
      "rewards/margins": 17.328655242919922,
      "rewards/real": -1.732965111732483,
      "step": 1010
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.931009957325747e-07,
      "logits/generated": -2.6576783657073975,
      "logits/real": -2.367976665496826,
      "logps/generated": -314.0473937988281,
      "logps/real": -135.40328979492188,
      "loss": 0.0043,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.228788375854492,
      "rewards/margins": 16.391712188720703,
      "rewards/real": -1.837074637413025,
      "step": 1020
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.895448079658606e-07,
      "logits/generated": -2.6377809047698975,
      "logits/real": -2.456001043319702,
      "logps/generated": -304.30194091796875,
      "logps/real": -158.57046508789062,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.988388061523438,
      "rewards/margins": 16.181509017944336,
      "rewards/real": -1.8068807125091553,
      "step": 1030
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.859886201991465e-07,
      "logits/generated": -2.713982343673706,
      "logits/real": -2.5400590896606445,
      "logps/generated": -319.47222900390625,
      "logps/real": -171.1156463623047,
      "loss": 0.004,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.150558471679688,
      "rewards/margins": 15.5328369140625,
      "rewards/real": -2.61772084236145,
      "step": 1040
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.8243243243243243e-07,
      "logits/generated": -2.6828341484069824,
      "logits/real": -2.451007604598999,
      "logps/generated": -323.1172790527344,
      "logps/real": -165.8328399658203,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.114700317382812,
      "rewards/margins": 16.214031219482422,
      "rewards/real": -2.900669813156128,
      "step": 1050
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.7887624466571835e-07,
      "logits/generated": -2.6970200538635254,
      "logits/real": -2.5064215660095215,
      "logps/generated": -334.9453125,
      "logps/real": -161.0951385498047,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.451995849609375,
      "rewards/margins": 16.473833084106445,
      "rewards/real": -2.9781641960144043,
      "step": 1060
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.7532005689900424e-07,
      "logits/generated": -2.680431842803955,
      "logits/real": -2.45689058303833,
      "logps/generated": -318.7887268066406,
      "logps/real": -160.87933349609375,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/generated": -18.85040855407715,
      "rewards/margins": 16.028974533081055,
      "rewards/real": -2.821434497833252,
      "step": 1070
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.717638691322902e-07,
      "logits/generated": -2.631726026535034,
      "logits/real": -2.430089235305786,
      "logps/generated": -325.7797546386719,
      "logps/real": -160.8832244873047,
      "loss": 0.005,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.576419830322266,
      "rewards/margins": 16.21274185180664,
      "rewards/real": -3.363679885864258,
      "step": 1080
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.6820768136557609e-07,
      "logits/generated": -2.6365928649902344,
      "logits/real": -2.410539388656616,
      "logps/generated": -320.75250244140625,
      "logps/real": -152.6123504638672,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.101604461669922,
      "rewards/margins": 16.658384323120117,
      "rewards/real": -2.4432196617126465,
      "step": 1090
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.64651493598862e-07,
      "logits/generated": -2.627570629119873,
      "logits/real": -2.4656529426574707,
      "logps/generated": -295.3066101074219,
      "logps/real": -164.18795776367188,
      "loss": 0.0041,
      "rewards/accuracies": 1.0,
      "rewards/generated": -17.335086822509766,
      "rewards/margins": 15.275428771972656,
      "rewards/real": -2.0596542358398438,
      "step": 1100
    },
    {
      "epoch": 0.7,
      "eval_logits/generated": -2.636800527572632,
      "eval_logits/real": -2.432948112487793,
      "eval_logps/generated": -320.8827209472656,
      "eval_logps/real": -159.4218292236328,
      "eval_loss": 0.005006026476621628,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -19.304651260375977,
      "eval_rewards/margins": 16.82903289794922,
      "eval_rewards/real": -2.4756181240081787,
      "eval_runtime": 358.0601,
      "eval_samples_per_second": 13.964,
      "eval_steps_per_second": 0.438,
      "step": 1100
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.6109530583214793e-07,
      "logits/generated": -2.6352360248565674,
      "logits/real": -2.3814148902893066,
      "logps/generated": -325.02911376953125,
      "logps/real": -155.18861389160156,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.062488555908203,
      "rewards/margins": 16.672191619873047,
      "rewards/real": -2.390298366546631,
      "step": 1110
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.5753911806543385e-07,
      "logits/generated": -2.5947158336639404,
      "logits/real": -2.370400905609131,
      "logps/generated": -328.9488220214844,
      "logps/real": -161.29063415527344,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.087726593017578,
      "rewards/margins": 17.328855514526367,
      "rewards/real": -2.7588706016540527,
      "step": 1120
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.5398293029871974e-07,
      "logits/generated": -2.5837631225585938,
      "logits/real": -2.3980956077575684,
      "logps/generated": -340.8889465332031,
      "logps/real": -171.73155212402344,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.824243545532227,
      "rewards/margins": 18.651147842407227,
      "rewards/real": -3.173096179962158,
      "step": 1130
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.504267425320057e-07,
      "logits/generated": -2.6092448234558105,
      "logits/real": -2.384089469909668,
      "logps/generated": -353.5443420410156,
      "logps/real": -164.05426025390625,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.162696838378906,
      "rewards/margins": 18.883724212646484,
      "rewards/real": -3.2789711952209473,
      "step": 1140
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.4687055476529158e-07,
      "logits/generated": -2.604997158050537,
      "logits/real": -2.4028961658477783,
      "logps/generated": -345.52691650390625,
      "logps/real": -178.81590270996094,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.2681827545166,
      "rewards/margins": 17.40066909790039,
      "rewards/real": -3.8675148487091064,
      "step": 1150
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.4331436699857753e-07,
      "logits/generated": -2.5864603519439697,
      "logits/real": -2.4394125938415527,
      "logps/generated": -358.2093200683594,
      "logps/real": -176.05535888671875,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.399456024169922,
      "rewards/margins": 18.71761703491211,
      "rewards/real": -3.6818385124206543,
      "step": 1160
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.3975817923186345e-07,
      "logits/generated": -2.622929096221924,
      "logits/real": -2.4487674236297607,
      "logps/generated": -359.487060546875,
      "logps/real": -182.25320434570312,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.96976089477539,
      "rewards/margins": 19.018558502197266,
      "rewards/real": -2.951202392578125,
      "step": 1170
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.3620199146514935e-07,
      "logits/generated": -2.5927932262420654,
      "logits/real": -2.4133658409118652,
      "logps/generated": -350.9914855957031,
      "logps/real": -157.60354614257812,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.829925537109375,
      "rewards/margins": 18.090991973876953,
      "rewards/real": -3.738931655883789,
      "step": 1180
    },
    {
      "epoch": 0.76,
      "learning_rate": 1.326458036984353e-07,
      "logits/generated": -2.6194968223571777,
      "logits/real": -2.442230463027954,
      "logps/generated": -328.1695556640625,
      "logps/real": -166.50439453125,
      "loss": 0.0101,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -20.360904693603516,
      "rewards/margins": 16.797086715698242,
      "rewards/real": -3.563816785812378,
      "step": 1190
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.290896159317212e-07,
      "logits/generated": -2.6396727561950684,
      "logits/real": -2.4529004096984863,
      "logps/generated": -351.5899658203125,
      "logps/real": -158.91500854492188,
      "loss": 0.0033,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.666501998901367,
      "rewards/margins": 18.48183822631836,
      "rewards/real": -3.1846625804901123,
      "step": 1200
    },
    {
      "epoch": 0.77,
      "eval_logits/generated": -2.62396502494812,
      "eval_logits/real": -2.4681177139282227,
      "eval_logps/generated": -330.46142578125,
      "eval_logps/real": -163.26539611816406,
      "eval_loss": 0.0037050044629722834,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -20.262523651123047,
      "eval_rewards/margins": 17.402549743652344,
      "eval_rewards/real": -2.8599753379821777,
      "eval_runtime": 353.2592,
      "eval_samples_per_second": 14.154,
      "eval_steps_per_second": 0.444,
      "step": 1200
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.255334281650071e-07,
      "logits/generated": -2.603823184967041,
      "logits/real": -2.468867540359497,
      "logps/generated": -342.6956787109375,
      "logps/real": -169.12026977539062,
      "loss": 0.0096,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -20.63424301147461,
      "rewards/margins": 17.795860290527344,
      "rewards/real": -2.8383822441101074,
      "step": 1210
    },
    {
      "epoch": 0.78,
      "learning_rate": 1.2197724039829303e-07,
      "logits/generated": -2.6127591133117676,
      "logits/real": -2.378269672393799,
      "logps/generated": -333.45458984375,
      "logps/real": -153.81045532226562,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.970136642456055,
      "rewards/margins": 17.384471893310547,
      "rewards/real": -2.5856640338897705,
      "step": 1220
    },
    {
      "epoch": 0.79,
      "learning_rate": 1.1842105263157894e-07,
      "logits/generated": -2.555279016494751,
      "logits/real": -2.393095016479492,
      "logps/generated": -334.4615478515625,
      "logps/real": -148.1243438720703,
      "loss": 0.0043,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.898094177246094,
      "rewards/margins": 17.973209381103516,
      "rewards/real": -2.9248833656311035,
      "step": 1230
    },
    {
      "epoch": 0.79,
      "learning_rate": 1.1486486486486487e-07,
      "logits/generated": -2.6060502529144287,
      "logits/real": -2.4234375953674316,
      "logps/generated": -325.2197570800781,
      "logps/real": -164.59268188476562,
      "loss": 0.0074,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.13725471496582,
      "rewards/margins": 17.43795394897461,
      "rewards/real": -2.6992993354797363,
      "step": 1240
    },
    {
      "epoch": 0.8,
      "learning_rate": 1.1130867709815078e-07,
      "logits/generated": -2.5801854133605957,
      "logits/real": -2.407710313796997,
      "logps/generated": -346.99639892578125,
      "logps/real": -163.65025329589844,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.688512802124023,
      "rewards/margins": 18.730518341064453,
      "rewards/real": -2.9579977989196777,
      "step": 1250
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.077524893314367e-07,
      "logits/generated": -2.569446086883545,
      "logits/real": -2.3492586612701416,
      "logps/generated": -362.0098571777344,
      "logps/real": -155.21597290039062,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.53670883178711,
      "rewards/margins": 19.49285316467285,
      "rewards/real": -3.043856143951416,
      "step": 1260
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.0419630156472262e-07,
      "logits/generated": -2.5712552070617676,
      "logits/real": -2.49360728263855,
      "logps/generated": -345.88848876953125,
      "logps/real": -187.5894775390625,
      "loss": 0.0026,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.826547622680664,
      "rewards/margins": 18.033016204833984,
      "rewards/real": -3.793531894683838,
      "step": 1270
    },
    {
      "epoch": 0.82,
      "learning_rate": 1.0064011379800854e-07,
      "logits/generated": -2.605649471282959,
      "logits/real": -2.4789321422576904,
      "logps/generated": -365.0675048828125,
      "logps/real": -179.69142150878906,
      "loss": 0.0059,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.86709213256836,
      "rewards/margins": 19.44388198852539,
      "rewards/real": -3.423208236694336,
      "step": 1280
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.708392603129445e-08,
      "logits/generated": -2.6148712635040283,
      "logits/real": -2.455827236175537,
      "logps/generated": -312.61138916015625,
      "logps/real": -163.89895629882812,
      "loss": 0.0043,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -18.890113830566406,
      "rewards/margins": 15.756782531738281,
      "rewards/real": -3.1333322525024414,
      "step": 1290
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.352773826458037e-08,
      "logits/generated": -2.61708927154541,
      "logits/real": -2.4340155124664307,
      "logps/generated": -358.7867126464844,
      "logps/real": -162.8968505859375,
      "loss": 0.0042,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.204402923583984,
      "rewards/margins": 18.589527130126953,
      "rewards/real": -2.614877223968506,
      "step": 1300
    },
    {
      "epoch": 0.83,
      "eval_logits/generated": -2.5973832607269287,
      "eval_logits/real": -2.4462833404541016,
      "eval_logps/generated": -335.50567626953125,
      "eval_logps/real": -161.40390014648438,
      "eval_loss": 0.003191521856933832,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -20.76694679260254,
      "eval_rewards/margins": 18.093122482299805,
      "eval_rewards/real": -2.673825740814209,
      "eval_runtime": 357.3744,
      "eval_samples_per_second": 13.991,
      "eval_steps_per_second": 0.439,
      "step": 1300
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.997155049786629e-08,
      "logits/generated": -2.6018290519714355,
      "logits/real": -2.384018659591675,
      "logps/generated": -346.05120849609375,
      "logps/real": -154.76527404785156,
      "loss": 0.006,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.367633819580078,
      "rewards/margins": 18.317487716674805,
      "rewards/real": -3.0501456260681152,
      "step": 1310
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.64153627311522e-08,
      "logits/generated": -2.591632604598999,
      "logits/real": -2.4401702880859375,
      "logps/generated": -376.4816589355469,
      "logps/real": -161.6133575439453,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/generated": -24.407400131225586,
      "rewards/margins": 21.652812957763672,
      "rewards/real": -2.7545878887176514,
      "step": 1320
    },
    {
      "epoch": 0.85,
      "learning_rate": 8.285917496443812e-08,
      "logits/generated": -2.614602565765381,
      "logits/real": -2.4444077014923096,
      "logps/generated": -353.73712158203125,
      "logps/real": -166.6267547607422,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.954816818237305,
      "rewards/margins": 19.12883758544922,
      "rewards/real": -2.825981378555298,
      "step": 1330
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.930298719772404e-08,
      "logits/generated": -2.584188461303711,
      "logits/real": -2.4484755992889404,
      "logps/generated": -343.3823547363281,
      "logps/real": -176.88575744628906,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.353042602539062,
      "rewards/margins": 18.155269622802734,
      "rewards/real": -3.1977739334106445,
      "step": 1340
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.574679943100994e-08,
      "logits/generated": -2.6062331199645996,
      "logits/real": -2.4200310707092285,
      "logps/generated": -346.0461120605469,
      "logps/real": -147.83078002929688,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.630901336669922,
      "rewards/margins": 18.697572708129883,
      "rewards/real": -2.9333269596099854,
      "step": 1350
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.219061166429587e-08,
      "logits/generated": -2.5920839309692383,
      "logits/real": -2.4139137268066406,
      "logps/generated": -330.27569580078125,
      "logps/real": -157.5228729248047,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.386878967285156,
      "rewards/margins": 17.642425537109375,
      "rewards/real": -2.7444536685943604,
      "step": 1360
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.863442389758179e-08,
      "logits/generated": -2.588639974594116,
      "logits/real": -2.4485459327697754,
      "logps/generated": -358.29486083984375,
      "logps/real": -170.88619995117188,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.756534576416016,
      "rewards/margins": 19.36898422241211,
      "rewards/real": -2.3875479698181152,
      "step": 1370
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.507823613086771e-08,
      "logits/generated": -2.616661548614502,
      "logits/real": -2.473728895187378,
      "logps/generated": -344.1673583984375,
      "logps/real": -171.72756958007812,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.39995574951172,
      "rewards/margins": 18.465641021728516,
      "rewards/real": -2.9343149662017822,
      "step": 1380
    },
    {
      "epoch": 0.89,
      "learning_rate": 6.152204836415363e-08,
      "logits/generated": -2.623744010925293,
      "logits/real": -2.4601359367370605,
      "logps/generated": -356.36572265625,
      "logps/real": -165.5369110107422,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.57801055908203,
      "rewards/margins": 18.742273330688477,
      "rewards/real": -2.8357362747192383,
      "step": 1390
    },
    {
      "epoch": 0.9,
      "learning_rate": 5.796586059743954e-08,
      "logits/generated": -2.58418345451355,
      "logits/real": -2.4554524421691895,
      "logps/generated": -342.3634338378906,
      "logps/real": -161.27557373046875,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.252824783325195,
      "rewards/margins": 18.831134796142578,
      "rewards/real": -2.4216885566711426,
      "step": 1400
    },
    {
      "epoch": 0.9,
      "eval_logits/generated": -2.6144187450408936,
      "eval_logits/real": -2.459505081176758,
      "eval_logps/generated": -334.29248046875,
      "eval_logps/real": -156.4322509765625,
      "eval_loss": 0.003032037289813161,
      "eval_rewards/accuracies": 0.9992038011550903,
      "eval_rewards/generated": -20.64562225341797,
      "eval_rewards/margins": 18.468963623046875,
      "eval_rewards/real": -2.1766605377197266,
      "eval_runtime": 353.8921,
      "eval_samples_per_second": 14.129,
      "eval_steps_per_second": 0.444,
      "step": 1400
    },
    {
      "epoch": 0.9,
      "learning_rate": 5.4409672830725456e-08,
      "logits/generated": -2.6259565353393555,
      "logits/real": -2.446854591369629,
      "logps/generated": -327.33294677734375,
      "logps/real": -157.96328735351562,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/generated": -19.752155303955078,
      "rewards/margins": 17.652597427368164,
      "rewards/real": -2.0995583534240723,
      "step": 1410
    },
    {
      "epoch": 0.91,
      "learning_rate": 5.0853485064011376e-08,
      "logits/generated": -2.6113741397857666,
      "logits/real": -2.3581416606903076,
      "logps/generated": -337.38336181640625,
      "logps/real": -144.90785217285156,
      "loss": 0.0075,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -20.918914794921875,
      "rewards/margins": 18.755264282226562,
      "rewards/real": -2.163649082183838,
      "step": 1420
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.72972972972973e-08,
      "logits/generated": -2.5806427001953125,
      "logits/real": -2.4139533042907715,
      "logps/generated": -328.04327392578125,
      "logps/real": -159.30142211914062,
      "loss": 0.0044,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.404008865356445,
      "rewards/margins": 18.004352569580078,
      "rewards/real": -2.3996574878692627,
      "step": 1430
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.374110953058322e-08,
      "logits/generated": -2.5779836177825928,
      "logits/real": -2.41233229637146,
      "logps/generated": -331.53558349609375,
      "logps/real": -157.76052856445312,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.964191436767578,
      "rewards/margins": 18.361709594726562,
      "rewards/real": -2.6024842262268066,
      "step": 1440
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.018492176386913e-08,
      "logits/generated": -2.5819339752197266,
      "logits/real": -2.4180991649627686,
      "logps/generated": -335.5973205566406,
      "logps/real": -161.56442260742188,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.901308059692383,
      "rewards/margins": 18.16925811767578,
      "rewards/real": -2.7320468425750732,
      "step": 1450
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.6628733997155046e-08,
      "logits/generated": -2.565924644470215,
      "logits/real": -2.343358278274536,
      "logps/generated": -367.3075256347656,
      "logps/real": -140.11587524414062,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/generated": -24.341123580932617,
      "rewards/margins": 21.56112289428711,
      "rewards/real": -2.7799980640411377,
      "step": 1460
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.3072546230440967e-08,
      "logits/generated": -2.5737624168395996,
      "logits/real": -2.3942620754241943,
      "logps/generated": -330.3951721191406,
      "logps/real": -156.66238403320312,
      "loss": 0.0037,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -20.762723922729492,
      "rewards/margins": 18.088510513305664,
      "rewards/real": -2.674212694168091,
      "step": 1470
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.9516358463726884e-08,
      "logits/generated": -2.5995941162109375,
      "logits/real": -2.4209532737731934,
      "logps/generated": -342.66656494140625,
      "logps/real": -166.10543823242188,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.383831024169922,
      "rewards/margins": 18.954071044921875,
      "rewards/real": -2.4297597408294678,
      "step": 1480
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.59601706970128e-08,
      "logits/generated": -2.5837836265563965,
      "logits/real": -2.4423680305480957,
      "logps/generated": -344.68670654296875,
      "logps/real": -168.974365234375,
      "loss": 0.0062,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.8641300201416,
      "rewards/margins": 19.55858612060547,
      "rewards/real": -2.3055408000946045,
      "step": 1490
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.240398293029872e-08,
      "logits/generated": -2.5561881065368652,
      "logits/real": -2.378162384033203,
      "logps/generated": -345.0732421875,
      "logps/real": -157.55331420898438,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.966922760009766,
      "rewards/margins": 19.243404388427734,
      "rewards/real": -2.7235145568847656,
      "step": 1500
    },
    {
      "epoch": 0.96,
      "eval_logits/generated": -2.5879950523376465,
      "eval_logits/real": -2.431485414505005,
      "eval_logps/generated": -346.59881591796875,
      "eval_logps/real": -161.42237854003906,
      "eval_loss": 0.002722462872043252,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/generated": -21.876266479492188,
      "eval_rewards/margins": 19.200592041015625,
      "eval_rewards/real": -2.6756742000579834,
      "eval_runtime": 358.839,
      "eval_samples_per_second": 13.934,
      "eval_steps_per_second": 0.438,
      "step": 1500
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.8847795163584636e-08,
      "logits/generated": -2.5720112323760986,
      "logits/real": -2.4139506816864014,
      "logps/generated": -347.1139221191406,
      "logps/real": -159.2230224609375,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.823551177978516,
      "rewards/margins": 19.27743911743164,
      "rewards/real": -2.546113967895508,
      "step": 1510
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5291607396870554e-08,
      "logits/generated": -2.59587025642395,
      "logits/real": -2.453565835952759,
      "logps/generated": -342.2265930175781,
      "logps/real": -155.0762481689453,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.84916877746582,
      "rewards/margins": 19.360273361206055,
      "rewards/real": -2.4888949394226074,
      "step": 1520
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.1735419630156473e-08,
      "logits/generated": -2.597951650619507,
      "logits/real": -2.443702459335327,
      "logps/generated": -356.6807556152344,
      "logps/real": -161.6763458251953,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.68019676208496,
      "rewards/margins": 19.755842208862305,
      "rewards/real": -2.9243524074554443,
      "step": 1530
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.179231863442388e-09,
      "logits/generated": -2.577643871307373,
      "logits/real": -2.454272985458374,
      "logps/generated": -328.8896484375,
      "logps/real": -161.75326538085938,
      "loss": 0.0066,
      "rewards/accuracies": 1.0,
      "rewards/generated": -20.900354385375977,
      "rewards/margins": 18.582683563232422,
      "rewards/real": -2.31767201423645,
      "step": 1540
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.623044096728307e-09,
      "logits/generated": -2.6040985584259033,
      "logits/real": -2.4326038360595703,
      "logps/generated": -353.8281555175781,
      "logps/real": -167.1890869140625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/generated": -22.231428146362305,
      "rewards/margins": 19.598224639892578,
      "rewards/real": -2.6332058906555176,
      "step": 1550
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.0668563300142248e-09,
      "logits/generated": -2.5751712322235107,
      "logits/real": -2.382331132888794,
      "logps/generated": -337.31011962890625,
      "logps/real": -157.5955047607422,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/generated": -21.168819427490234,
      "rewards/margins": 18.588611602783203,
      "rewards/real": -2.5802083015441895,
      "step": 1560
    },
    {
      "epoch": 1.0,
      "step": 1563,
      "total_flos": 0.0,
      "train_loss": 0.016542167173306324,
      "train_runtime": 14061.5551,
      "train_samples_per_second": 3.556,
      "train_steps_per_second": 0.111
    }
  ],
  "logging_steps": 10,
  "max_steps": 1563,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}