approx_nash_again_iter_2 / trainer_state.json
YYYYYYibo's picture
Model save
87b992d verified
raw
history blame
11 kB
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9984,
"eval_steps": 500,
"global_step": 156,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.01,
"eta": 0.0010000000474974513,
"grad_norm": 14.26872732684494,
"learning_rate": 3.125e-08,
"logits/chosen": -2.0110533237457275,
"logits/rejected": -2.249422550201416,
"logps/chosen": -141.09288024902344,
"logps/pi_response": -121.86016845703125,
"logps/ref_response": -121.86016845703125,
"logps/rejected": -141.46664428710938,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.06,
"eta": 0.0010000000474974513,
"grad_norm": 17.697699996792654,
"learning_rate": 3.1249999999999997e-07,
"logits/chosen": -2.2928249835968018,
"logits/rejected": -2.312570810317993,
"logps/chosen": -172.20089721679688,
"logps/pi_response": -145.86224365234375,
"logps/ref_response": -144.9134979248047,
"logps/rejected": -177.94036865234375,
"loss": 0.6924,
"rewards/accuracies": 0.40625,
"rewards/chosen": -0.007550257723778486,
"rewards/margins": 0.0005382309318520129,
"rewards/rejected": -0.008088488131761551,
"step": 10
},
{
"epoch": 0.13,
"eta": 0.0010000000474974513,
"grad_norm": 19.324417294835,
"learning_rate": 4.989935734988097e-07,
"logits/chosen": -2.3221993446350098,
"logits/rejected": -2.2393600940704346,
"logps/chosen": -194.87570190429688,
"logps/pi_response": -169.71463012695312,
"logps/ref_response": -142.37953186035156,
"logps/rejected": -202.0398712158203,
"loss": 0.6957,
"rewards/accuracies": 0.46875,
"rewards/chosen": -0.2509865462779999,
"rewards/margins": 0.005367345176637173,
"rewards/rejected": -0.25635388493537903,
"step": 20
},
{
"epoch": 0.19,
"eta": 0.0010000000474974513,
"grad_norm": 22.196329662655977,
"learning_rate": 4.877641290737883e-07,
"logits/chosen": -2.3096923828125,
"logits/rejected": -2.27471923828125,
"logps/chosen": -206.6990966796875,
"logps/pi_response": -181.54678344726562,
"logps/ref_response": -146.62493896484375,
"logps/rejected": -208.9296112060547,
"loss": 0.6937,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.3044261336326599,
"rewards/margins": 0.00788620300590992,
"rewards/rejected": -0.3123123049736023,
"step": 30
},
{
"epoch": 0.26,
"eta": 0.0010000000474974513,
"grad_norm": 17.53383943563289,
"learning_rate": 4.646121984004665e-07,
"logits/chosen": -2.53867244720459,
"logits/rejected": -2.4202938079833984,
"logps/chosen": -176.26608276367188,
"logps/pi_response": -150.63552856445312,
"logps/ref_response": -143.65682983398438,
"logps/rejected": -177.16404724121094,
"loss": 0.6915,
"rewards/accuracies": 0.543749988079071,
"rewards/chosen": -0.06383548676967621,
"rewards/margins": 0.021249257028102875,
"rewards/rejected": -0.08508473634719849,
"step": 40
},
{
"epoch": 0.32,
"eta": 0.0010000000474974513,
"grad_norm": 22.495476546149213,
"learning_rate": 4.3069871595684787e-07,
"logits/chosen": -2.456916570663452,
"logits/rejected": -2.314687728881836,
"logps/chosen": -193.93612670898438,
"logps/pi_response": -164.81993103027344,
"logps/ref_response": -150.46957397460938,
"logps/rejected": -194.20321655273438,
"loss": 0.6832,
"rewards/accuracies": 0.5093749761581421,
"rewards/chosen": -0.10493893921375275,
"rewards/margins": 0.04571269080042839,
"rewards/rejected": -0.15065164864063263,
"step": 50
},
{
"epoch": 0.38,
"eta": 0.0010000000474974513,
"grad_norm": 16.374382979297494,
"learning_rate": 3.877242453630256e-07,
"logits/chosen": -2.306607484817505,
"logits/rejected": -2.2804460525512695,
"logps/chosen": -239.197998046875,
"logps/pi_response": -211.57894897460938,
"logps/ref_response": -151.56642150878906,
"logps/rejected": -241.3532257080078,
"loss": 0.699,
"rewards/accuracies": 0.546875,
"rewards/chosen": -0.5582221150398254,
"rewards/margins": 0.025322098284959793,
"rewards/rejected": -0.583544135093689,
"step": 60
},
{
"epoch": 0.45,
"eta": 0.0010000000474974513,
"grad_norm": 14.529411357126248,
"learning_rate": 3.378437060203357e-07,
"logits/chosen": -2.3658206462860107,
"logits/rejected": -2.25939679145813,
"logps/chosen": -212.8553009033203,
"logps/pi_response": -183.07717895507812,
"logps/ref_response": -146.43084716796875,
"logps/rejected": -210.58859252929688,
"loss": 0.689,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.3529232144355774,
"rewards/margins": 0.01966114342212677,
"rewards/rejected": -0.37258434295654297,
"step": 70
},
{
"epoch": 0.51,
"eta": 0.0010000000474974513,
"grad_norm": 16.20000732293231,
"learning_rate": 2.8355831645441387e-07,
"logits/chosen": -2.3814728260040283,
"logits/rejected": -2.416779041290283,
"logps/chosen": -198.688232421875,
"logps/pi_response": -169.83807373046875,
"logps/ref_response": -141.36282348632812,
"logps/rejected": -201.0040740966797,
"loss": 0.6799,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -0.26767629384994507,
"rewards/margins": 0.015592445619404316,
"rewards/rejected": -0.2832687199115753,
"step": 80
},
{
"epoch": 0.58,
"eta": 0.0010000000474974513,
"grad_norm": 27.20849020997422,
"learning_rate": 2.2759017277414164e-07,
"logits/chosen": -2.282968044281006,
"logits/rejected": -2.3503777980804443,
"logps/chosen": -211.1111297607422,
"logps/pi_response": -184.20970153808594,
"logps/ref_response": -149.5955352783203,
"logps/rejected": -220.11575317382812,
"loss": 0.6789,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.34969550371170044,
"rewards/margins": 0.02153094671666622,
"rewards/rejected": -0.3712264895439148,
"step": 90
},
{
"epoch": 0.64,
"eta": 0.0010000000474974513,
"grad_norm": 16.29698307520508,
"learning_rate": 1.7274575140626315e-07,
"logits/chosen": -2.320495367050171,
"logits/rejected": -2.224569797515869,
"logps/chosen": -212.6381378173828,
"logps/pi_response": -183.83807373046875,
"logps/ref_response": -140.60658264160156,
"logps/rejected": -214.096435546875,
"loss": 0.6853,
"rewards/accuracies": 0.578125,
"rewards/chosen": -0.41716060042381287,
"rewards/margins": 0.0342952124774456,
"rewards/rejected": -0.45145583152770996,
"step": 100
},
{
"epoch": 0.7,
"eta": 0.0010000000474974513,
"grad_norm": 19.009812577905546,
"learning_rate": 1.2177518064852348e-07,
"logits/chosen": -2.3668313026428223,
"logits/rejected": -2.2519187927246094,
"logps/chosen": -211.2959442138672,
"logps/pi_response": -180.33094787597656,
"logps/ref_response": -139.77435302734375,
"logps/rejected": -211.1301727294922,
"loss": 0.6808,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -0.36967089772224426,
"rewards/margins": 0.052646201103925705,
"rewards/rejected": -0.42231711745262146,
"step": 110
},
{
"epoch": 0.77,
"eta": 0.0010000000474974513,
"grad_norm": 19.269930008011357,
"learning_rate": 7.723433775328384e-08,
"logits/chosen": -2.306391716003418,
"logits/rejected": -2.3454627990722656,
"logps/chosen": -211.0600128173828,
"logps/pi_response": -182.8990936279297,
"logps/ref_response": -149.42483520507812,
"logps/rejected": -218.762939453125,
"loss": 0.6813,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.2972378134727478,
"rewards/margins": 0.06230615824460983,
"rewards/rejected": -0.35954394936561584,
"step": 120
},
{
"epoch": 0.83,
"eta": 0.0010000000474974513,
"grad_norm": 21.30435634566315,
"learning_rate": 4.1356686569674335e-08,
"logits/chosen": -2.2520415782928467,
"logits/rejected": -2.2255797386169434,
"logps/chosen": -204.5604248046875,
"logps/pi_response": -179.6775665283203,
"logps/ref_response": -142.9306640625,
"logps/rejected": -207.15036010742188,
"loss": 0.6781,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.336225688457489,
"rewards/margins": 0.039131637662649155,
"rewards/rejected": -0.37535732984542847,
"step": 130
},
{
"epoch": 0.9,
"eta": 0.0010000000474974513,
"grad_norm": 17.88461135981544,
"learning_rate": 1.5941282340065697e-08,
"logits/chosen": -2.3840537071228027,
"logits/rejected": -2.356271982192993,
"logps/chosen": -196.31777954101562,
"logps/pi_response": -173.85084533691406,
"logps/ref_response": -141.44508361816406,
"logps/rejected": -209.9248046875,
"loss": 0.6732,
"rewards/accuracies": 0.596875011920929,
"rewards/chosen": -0.3001561164855957,
"rewards/margins": 0.08288339525461197,
"rewards/rejected": -0.38303953409194946,
"step": 140
},
{
"epoch": 0.96,
"eta": 0.0010000000474974513,
"grad_norm": 21.25233665353108,
"learning_rate": 2.2625595580163247e-09,
"logits/chosen": -2.219001293182373,
"logits/rejected": -2.2412266731262207,
"logps/chosen": -209.44009399414062,
"logps/pi_response": -184.37173461914062,
"logps/ref_response": -143.34500122070312,
"logps/rejected": -216.7500457763672,
"loss": 0.6747,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.3745076060295105,
"rewards/margins": 0.06318662315607071,
"rewards/rejected": -0.4376942217350006,
"step": 150
},
{
"epoch": 1.0,
"step": 156,
"total_flos": 0.0,
"train_loss": 0.6841066911434516,
"train_runtime": 31793.1415,
"train_samples_per_second": 0.629,
"train_steps_per_second": 0.005
}
],
"logging_steps": 10,
"max_steps": 156,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 200,
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}