{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9968652037617555,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 3.125e-08,
      "logits/chosen": -1.995415210723877,
      "logits/rejected": -2.0361223220825195,
      "logps/chosen": -475.42401123046875,
      "logps/pi_response": -236.53262329101562,
      "logps/ref_response": -236.53262329101562,
      "logps/rejected": -571.512451171875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.009598970413208,
      "logits/rejected": -1.9492552280426025,
      "logps/chosen": -398.6226501464844,
      "logps/pi_response": -160.84869384765625,
      "logps/ref_response": -161.05593872070312,
      "logps/rejected": -436.8831481933594,
      "loss": 0.6907,
      "rewards/accuracies": 0.4652777910232544,
      "rewards/chosen": -0.00939179491251707,
      "rewards/margins": 0.001390137942507863,
      "rewards/rejected": -0.010781930759549141,
      "step": 10
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -1.9771480560302734,
      "logits/rejected": -1.934857726097107,
      "logps/chosen": -397.58258056640625,
      "logps/pi_response": -166.76702880859375,
      "logps/ref_response": -170.16830444335938,
      "logps/rejected": -543.4985961914062,
      "loss": 0.647,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2936175465583801,
      "rewards/margins": 0.1937868893146515,
      "rewards/rejected": -0.4874044954776764,
      "step": 20
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -1.9177439212799072,
      "logits/rejected": -1.8892943859100342,
      "logps/chosen": -484.595947265625,
      "logps/pi_response": -180.4770965576172,
      "logps/ref_response": -176.73623657226562,
      "logps/rejected": -564.6936645507812,
      "loss": 0.6744,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.7892134785652161,
      "rewards/margins": 0.32036879658699036,
      "rewards/rejected": -1.1095821857452393,
      "step": 30
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -1.9668203592300415,
      "logits/rejected": -1.9143187999725342,
      "logps/chosen": -483.791015625,
      "logps/pi_response": -200.40647888183594,
      "logps/ref_response": -175.1200408935547,
      "logps/rejected": -604.1815185546875,
      "loss": 0.6401,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7993307113647461,
      "rewards/margins": 0.2609265446662903,
      "rewards/rejected": -1.0602573156356812,
      "step": 40
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -1.839737892150879,
      "logits/rejected": -1.6476318836212158,
      "logps/chosen": -528.5265502929688,
      "logps/pi_response": -202.95977783203125,
      "logps/ref_response": -184.63198852539062,
      "logps/rejected": -655.3939208984375,
      "loss": 0.5966,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.9922237396240234,
      "rewards/margins": 0.4673174321651459,
      "rewards/rejected": -1.4595410823822021,
      "step": 50
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -1.1266146898269653,
      "logits/rejected": -0.9666005969047546,
      "logps/chosen": -499.87744140625,
      "logps/pi_response": -201.53793334960938,
      "logps/ref_response": -175.98574829101562,
      "logps/rejected": -626.661376953125,
      "loss": 0.5888,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.8994181752204895,
      "rewards/margins": 0.5771986842155457,
      "rewards/rejected": -1.4766168594360352,
      "step": 60
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -0.5178133845329285,
      "logits/rejected": -0.19050300121307373,
      "logps/chosen": -519.18115234375,
      "logps/pi_response": -228.1885528564453,
      "logps/ref_response": -161.18075561523438,
      "logps/rejected": -662.8558349609375,
      "loss": 0.586,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.2691316604614258,
      "rewards/margins": 0.6006889939308167,
      "rewards/rejected": -1.8698203563690186,
      "step": 70
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -0.6555771827697754,
      "logits/rejected": -0.3421247899532318,
      "logps/chosen": -536.9698486328125,
      "logps/pi_response": -224.0596923828125,
      "logps/ref_response": -175.85252380371094,
      "logps/rejected": -673.0121459960938,
      "loss": 0.5367,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2228807210922241,
      "rewards/margins": 0.626615583896637,
      "rewards/rejected": -1.8494962453842163,
      "step": 80
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -0.19603128731250763,
      "logits/rejected": 0.2836986184120178,
      "logps/chosen": -586.1405029296875,
      "logps/pi_response": -253.3416290283203,
      "logps/ref_response": -184.4833526611328,
      "logps/rejected": -720.2657470703125,
      "loss": 0.5611,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.6138103008270264,
      "rewards/margins": 0.6346837878227234,
      "rewards/rejected": -2.2484939098358154,
      "step": 90
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -0.19370171427726746,
      "logits/rejected": 0.2344537228345871,
      "logps/chosen": -579.291015625,
      "logps/pi_response": -266.8283386230469,
      "logps/ref_response": -191.95008850097656,
      "logps/rejected": -695.4265747070312,
      "loss": 0.5654,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.5290648937225342,
      "rewards/margins": 0.5712359547615051,
      "rewards/rejected": -2.1003005504608154,
      "step": 100
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -0.3242037892341614,
      "logits/rejected": 0.11515514552593231,
      "logps/chosen": -574.2125854492188,
      "logps/pi_response": -237.9784698486328,
      "logps/ref_response": -172.76661682128906,
      "logps/rejected": -651.3258056640625,
      "loss": 0.5715,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.267407774925232,
      "rewards/margins": 0.5044547915458679,
      "rewards/rejected": -1.7718626260757446,
      "step": 110
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -0.09869599342346191,
      "logits/rejected": 0.1822367161512375,
      "logps/chosen": -539.43701171875,
      "logps/pi_response": -243.4367218017578,
      "logps/ref_response": -166.47335815429688,
      "logps/rejected": -692.1175537109375,
      "loss": 0.5311,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.3617780208587646,
      "rewards/margins": 0.6481711268424988,
      "rewards/rejected": -2.009949207305908,
      "step": 120
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -0.07027504593133926,
      "logits/rejected": 0.25386515259742737,
      "logps/chosen": -550.7713623046875,
      "logps/pi_response": -254.2024383544922,
      "logps/ref_response": -178.47569274902344,
      "logps/rejected": -677.2970581054688,
      "loss": 0.5504,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3074489831924438,
      "rewards/margins": 0.5863579511642456,
      "rewards/rejected": -1.8938068151474,
      "step": 130
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 0.026742279529571533,
      "logits/rejected": 0.45248135924339294,
      "logps/chosen": -533.3641357421875,
      "logps/pi_response": -244.9961700439453,
      "logps/ref_response": -176.0931396484375,
      "logps/rejected": -698.1570434570312,
      "loss": 0.5502,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3009979724884033,
      "rewards/margins": 0.6849142909049988,
      "rewards/rejected": -1.9859119653701782,
      "step": 140
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -0.41795143485069275,
      "logits/rejected": 0.16397379338741302,
      "logps/chosen": -548.3402099609375,
      "logps/pi_response": -245.32962036132812,
      "logps/ref_response": -180.5318145751953,
      "logps/rejected": -688.0223999023438,
      "loss": 0.5526,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.2109647989273071,
      "rewards/margins": 0.7583649754524231,
      "rewards/rejected": -1.969329833984375,
      "step": 150
    },
    {
      "epoch": 1.0,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.5857575074681696,
      "train_runtime": 4237.6593,
      "train_samples_per_second": 4.809,
      "train_steps_per_second": 0.038
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}