| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 73, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 29.326078015981345, | |
| "learning_rate": 6.25e-08, | |
| "logps/chosen": -47.87165832519531, | |
| "logps/rejected": -35.03704071044922, | |
| "loss": 0.6939, | |
| "losses/dpo": 0.7437427639961243, | |
| "losses/sft": 0.2519839406013489, | |
| "losses/total": 0.7437427639961243, | |
| "ref_logps/chosen": -47.90069580078125, | |
| "ref_logps/rejected": -35.07575225830078, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.0029037208296358585, | |
| "rewards/margins": -0.0009674869943410158, | |
| "rewards/rejected": 0.0038712075911462307, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 25.98987817588094, | |
| "learning_rate": 1.25e-07, | |
| "logps/chosen": -46.03837966918945, | |
| "logps/rejected": -34.79166030883789, | |
| "loss": 0.6937, | |
| "losses/dpo": 0.711306095123291, | |
| "losses/sft": 0.21511156857013702, | |
| "losses/total": 0.711306095123291, | |
| "ref_logps/chosen": -46.05853271484375, | |
| "ref_logps/rejected": -34.81706237792969, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0020157406106591225, | |
| "rewards/margins": -0.000524366507306695, | |
| "rewards/rejected": 0.002540107350796461, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 43.145173675858224, | |
| "learning_rate": 1.875e-07, | |
| "logps/chosen": -41.797569274902344, | |
| "logps/rejected": -31.708539962768555, | |
| "loss": 0.693, | |
| "losses/dpo": 0.7042351365089417, | |
| "losses/sft": 0.18763618171215057, | |
| "losses/total": 0.7042351365089417, | |
| "ref_logps/chosen": -41.833030700683594, | |
| "ref_logps/rejected": -31.735107421875, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.003545756684616208, | |
| "rewards/margins": 0.0008889732416719198, | |
| "rewards/rejected": 0.0026567834429442883, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 31.32790996670384, | |
| "learning_rate": 2.5e-07, | |
| "logps/chosen": -42.71172332763672, | |
| "logps/rejected": -32.757808685302734, | |
| "loss": 0.6927, | |
| "losses/dpo": 0.6976655125617981, | |
| "losses/sft": 0.17784112691879272, | |
| "losses/total": 0.6976655125617981, | |
| "ref_logps/chosen": -42.72623062133789, | |
| "ref_logps/rejected": -32.75667190551758, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0014508566819131374, | |
| "rewards/margins": 0.0015643269289284945, | |
| "rewards/rejected": -0.00011346983956173062, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 35.10577986645193, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logps/chosen": -45.85194396972656, | |
| "logps/rejected": -34.628639221191406, | |
| "loss": 0.689, | |
| "losses/dpo": 0.7395577430725098, | |
| "losses/sft": 0.17383158206939697, | |
| "losses/total": 0.7395577430725098, | |
| "ref_logps/chosen": -45.91680145263672, | |
| "ref_logps/rejected": -34.60468673706055, | |
| "rewards/accuracies": 0.5703125, | |
| "rewards/chosen": 0.006485694088041782, | |
| "rewards/margins": 0.008881103247404099, | |
| "rewards/rejected": -0.002395408693701029, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 28.260278751569523, | |
| "learning_rate": 3.75e-07, | |
| "logps/chosen": -42.09749221801758, | |
| "logps/rejected": -32.70561599731445, | |
| "loss": 0.6932, | |
| "losses/dpo": 0.6590798497200012, | |
| "losses/sft": 0.18368251621723175, | |
| "losses/total": 0.6590798497200012, | |
| "ref_logps/chosen": -42.06741714477539, | |
| "ref_logps/rejected": -32.67097473144531, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": -0.0030076471157372, | |
| "rewards/margins": 0.0004564363043755293, | |
| "rewards/rejected": -0.0034640836529433727, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 45.257780534421805, | |
| "learning_rate": 4.375e-07, | |
| "logps/chosen": -48.16801834106445, | |
| "logps/rejected": -35.98320770263672, | |
| "loss": 0.6931, | |
| "losses/dpo": 0.674820065498352, | |
| "losses/sft": 0.17130310833454132, | |
| "losses/total": 0.674820065498352, | |
| "ref_logps/chosen": -48.16166687011719, | |
| "ref_logps/rejected": -35.96845245361328, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -0.0006352070486173034, | |
| "rewards/margins": 0.0008399828802794218, | |
| "rewards/rejected": -0.0014751903945580125, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 37.963707614132204, | |
| "learning_rate": 5e-07, | |
| "logps/chosen": -46.631561279296875, | |
| "logps/rejected": -34.54258728027344, | |
| "loss": 0.6911, | |
| "losses/dpo": 0.6616916060447693, | |
| "losses/sft": 0.15279927849769592, | |
| "losses/total": 0.6616916060447693, | |
| "ref_logps/chosen": -46.690643310546875, | |
| "ref_logps/rejected": -34.551368713378906, | |
| "rewards/accuracies": 0.5546875, | |
| "rewards/chosen": 0.005908225197345018, | |
| "rewards/margins": 0.005030112341046333, | |
| "rewards/rejected": 0.0008781132637523115, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 23.24345634411509, | |
| "learning_rate": 4.997080567080816e-07, | |
| "logps/chosen": -45.053184509277344, | |
| "logps/rejected": -35.14673614501953, | |
| "loss": 0.6888, | |
| "losses/dpo": 0.645126461982727, | |
| "losses/sft": 0.1863231658935547, | |
| "losses/total": 0.645126461982727, | |
| "ref_logps/chosen": -45.13517379760742, | |
| "ref_logps/rejected": -35.132957458496094, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.008199075236916542, | |
| "rewards/margins": 0.009576688520610332, | |
| "rewards/rejected": -0.0013776118867099285, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 27.949597341892236, | |
| "learning_rate": 4.988329086794122e-07, | |
| "logps/chosen": -46.718475341796875, | |
| "logps/rejected": -36.01044464111328, | |
| "loss": 0.6845, | |
| "losses/dpo": 0.6536989212036133, | |
| "losses/sft": 0.16235677897930145, | |
| "losses/total": 0.6536989212036133, | |
| "ref_logps/chosen": -46.86553192138672, | |
| "ref_logps/rejected": -35.97478103637695, | |
| "rewards/accuracies": 0.6171875, | |
| "rewards/chosen": 0.0147053562104702, | |
| "rewards/margins": 0.018271632492542267, | |
| "rewards/rejected": -0.0035662769805639982, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 40.316536183472955, | |
| "learning_rate": 4.973765998627628e-07, | |
| "logps/chosen": -45.7076416015625, | |
| "logps/rejected": -32.744361877441406, | |
| "loss": 0.6758, | |
| "losses/dpo": 0.639275848865509, | |
| "losses/sft": 0.19072100520133972, | |
| "losses/total": 0.639275848865509, | |
| "ref_logps/chosen": -45.953941345214844, | |
| "ref_logps/rejected": -32.63063430786133, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": 0.024630192667245865, | |
| "rewards/margins": 0.036002762615680695, | |
| "rewards/rejected": -0.01137256994843483, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 31.231333750699285, | |
| "learning_rate": 4.953425315348533e-07, | |
| "logps/chosen": -48.346229553222656, | |
| "logps/rejected": -35.44029235839844, | |
| "loss": 0.6735, | |
| "losses/dpo": 0.7411879301071167, | |
| "losses/sft": 0.30462783575057983, | |
| "losses/total": 0.7411879301071167, | |
| "ref_logps/chosen": -48.579471588134766, | |
| "ref_logps/rejected": -35.26258087158203, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.023324450477957726, | |
| "rewards/margins": 0.04109576344490051, | |
| "rewards/rejected": -0.017771316692233086, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 24.02378939332813, | |
| "learning_rate": 4.92735454356513e-07, | |
| "logps/chosen": -43.760799407958984, | |
| "logps/rejected": -32.20792007446289, | |
| "loss": 0.6771, | |
| "losses/dpo": 0.7643380761146545, | |
| "losses/sft": 0.15294401347637177, | |
| "losses/total": 0.7643380761146545, | |
| "ref_logps/chosen": -43.909759521484375, | |
| "ref_logps/rejected": -32.016273498535156, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.01489595789462328, | |
| "rewards/margins": 0.034060731530189514, | |
| "rewards/rejected": -0.01916477642953396, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 33.47814491109199, | |
| "learning_rate": 4.895614572772916e-07, | |
| "logps/chosen": -45.79880905151367, | |
| "logps/rejected": -34.85653305053711, | |
| "loss": 0.6669, | |
| "losses/dpo": 0.7224411368370056, | |
| "losses/sft": 0.2095840573310852, | |
| "losses/total": 0.7224411368370056, | |
| "ref_logps/chosen": -46.07813262939453, | |
| "ref_logps/rejected": -34.58377456665039, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": 0.02793230675160885, | |
| "rewards/margins": 0.055208105593919754, | |
| "rewards/rejected": -0.027275800704956055, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 47.78782257013143, | |
| "learning_rate": 4.858279533144357e-07, | |
| "logps/chosen": -47.91066360473633, | |
| "logps/rejected": -36.8038330078125, | |
| "loss": 0.6545, | |
| "losses/dpo": 0.5712046027183533, | |
| "losses/sft": 0.20200778543949127, | |
| "losses/total": 0.5712046027183533, | |
| "ref_logps/chosen": -48.32217788696289, | |
| "ref_logps/rejected": -36.395023345947266, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": 0.04115153104066849, | |
| "rewards/margins": 0.08203274011611938, | |
| "rewards/rejected": -0.040881212800741196, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 246.97737804069968, | |
| "learning_rate": 4.815436622394441e-07, | |
| "logps/chosen": -46.90559387207031, | |
| "logps/rejected": -36.626888275146484, | |
| "loss": 0.6465, | |
| "losses/dpo": 0.7274478077888489, | |
| "losses/sft": 0.26765260100364685, | |
| "losses/total": 0.7274478077888489, | |
| "ref_logps/chosen": -47.21229934692383, | |
| "ref_logps/rejected": -35.93655776977539, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": 0.03067046031355858, | |
| "rewards/margins": 0.09970355033874512, | |
| "rewards/rejected": -0.06903309375047684, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 23.079239827774252, | |
| "learning_rate": 4.767185902126363e-07, | |
| "logps/chosen": -48.87858200073242, | |
| "logps/rejected": -36.90644073486328, | |
| "loss": 0.633, | |
| "losses/dpo": 0.6357161998748779, | |
| "losses/sft": 0.1839471459388733, | |
| "losses/total": 0.6357161998748779, | |
| "ref_logps/chosen": -49.40204620361328, | |
| "ref_logps/rejected": -36.11450958251953, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": 0.05234625190496445, | |
| "rewards/margins": 0.13153919577598572, | |
| "rewards/rejected": -0.07919295132160187, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 25.63300252359878, | |
| "learning_rate": 4.7136400641330245e-07, | |
| "logps/chosen": -46.71650695800781, | |
| "logps/rejected": -37.09510040283203, | |
| "loss": 0.6297, | |
| "losses/dpo": 0.6393631100654602, | |
| "losses/sft": 0.21227942407131195, | |
| "losses/total": 0.6393631100654602, | |
| "ref_logps/chosen": -46.991477966308594, | |
| "ref_logps/rejected": -35.969173431396484, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": 0.02749716117978096, | |
| "rewards/margins": 0.14008952677249908, | |
| "rewards/rejected": -0.11259236931800842, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 26.311859157755837, | |
| "learning_rate": 4.6549241672001225e-07, | |
| "logps/chosen": -43.63357162475586, | |
| "logps/rejected": -34.979026794433594, | |
| "loss": 0.6077, | |
| "losses/dpo": 0.5548383593559265, | |
| "losses/sft": 0.19493867456912994, | |
| "losses/total": 0.5548383593559265, | |
| "ref_logps/chosen": -44.03193664550781, | |
| "ref_logps/rejected": -33.485252380371094, | |
| "rewards/accuracies": 0.8515625, | |
| "rewards/chosen": 0.03983645513653755, | |
| "rewards/margins": 0.18921390175819397, | |
| "rewards/rejected": -0.14937745034694672, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 28.714173620781665, | |
| "learning_rate": 4.591175345025566e-07, | |
| "logps/chosen": -46.371559143066406, | |
| "logps/rejected": -35.243812561035156, | |
| "loss": 0.609, | |
| "losses/dpo": 0.6410955190658569, | |
| "losses/sft": 0.16183941066265106, | |
| "losses/total": 0.6410955190658569, | |
| "ref_logps/chosen": -46.70909881591797, | |
| "ref_logps/rejected": -33.71453857421875, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": 0.03375420719385147, | |
| "rewards/margins": 0.18668171763420105, | |
| "rewards/rejected": -0.15292751789093018, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 26.549036618365495, | |
| "learning_rate": 4.5225424859373684e-07, | |
| "logps/chosen": -41.521549224853516, | |
| "logps/rejected": -34.770103454589844, | |
| "loss": 0.5963, | |
| "losses/dpo": 0.7364767789840698, | |
| "losses/sft": 0.17622552812099457, | |
| "losses/total": 0.7364767789840698, | |
| "ref_logps/chosen": -41.7501106262207, | |
| "ref_logps/rejected": -32.80527114868164, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": 0.02285606414079666, | |
| "rewards/margins": 0.21933907270431519, | |
| "rewards/rejected": -0.19648301601409912, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 33.26960463303905, | |
| "learning_rate": 4.4491858851580553e-07, | |
| "logps/chosen": -45.94141387939453, | |
| "logps/rejected": -36.16654968261719, | |
| "loss": 0.5887, | |
| "losses/dpo": 0.495862752199173, | |
| "losses/sft": 0.17526012659072876, | |
| "losses/total": 0.495862752199173, | |
| "ref_logps/chosen": -46.16797637939453, | |
| "ref_logps/rejected": -33.92024612426758, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": 0.02265631966292858, | |
| "rewards/margins": 0.2472866028547287, | |
| "rewards/rejected": -0.22463028132915497, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 38.94504011639214, | |
| "learning_rate": 4.3712768704277524e-07, | |
| "logps/chosen": -43.17596435546875, | |
| "logps/rejected": -35.83791732788086, | |
| "loss": 0.5549, | |
| "losses/dpo": 0.6368575692176819, | |
| "losses/sft": 0.20419813692569733, | |
| "losses/total": 0.6368575692176819, | |
| "ref_logps/chosen": -43.439910888671875, | |
| "ref_logps/rejected": -32.738441467285156, | |
| "rewards/accuracies": 0.8828125, | |
| "rewards/chosen": 0.026394736021757126, | |
| "rewards/margins": 0.3363422751426697, | |
| "rewards/rejected": -0.30994755029678345, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 28.33928817647071, | |
| "learning_rate": 4.2889974018603024e-07, | |
| "logps/chosen": -48.73534393310547, | |
| "logps/rejected": -40.98769760131836, | |
| "loss": 0.5358, | |
| "losses/dpo": 0.6388107538223267, | |
| "losses/sft": 0.21662825345993042, | |
| "losses/total": 0.6388107538223267, | |
| "ref_logps/chosen": -48.840187072753906, | |
| "ref_logps/rejected": -37.24340057373047, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": 0.010484418831765652, | |
| "rewards/margins": 0.38491398096084595, | |
| "rewards/rejected": -0.3744295537471771, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 31.571769897086057, | |
| "learning_rate": 4.2025396469669926e-07, | |
| "logps/chosen": -49.65196228027344, | |
| "logps/rejected": -39.15043258666992, | |
| "loss": 0.5317, | |
| "losses/dpo": 0.4821869134902954, | |
| "losses/sft": 0.2129327803850174, | |
| "losses/total": 0.4821869134902954, | |
| "ref_logps/chosen": -49.09580993652344, | |
| "ref_logps/rejected": -34.47374725341797, | |
| "rewards/accuracies": 0.8828125, | |
| "rewards/chosen": -0.05561504885554314, | |
| "rewards/margins": 0.41205331683158875, | |
| "rewards/rejected": -0.467668354511261, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 20.54896163205101, | |
| "learning_rate": 4.112105531840426e-07, | |
| "logps/chosen": -50.22370529174805, | |
| "logps/rejected": -38.49211120605469, | |
| "loss": 0.5133, | |
| "losses/dpo": 0.6953214406967163, | |
| "losses/sft": 0.1770307421684265, | |
| "losses/total": 0.6953214406967163, | |
| "ref_logps/chosen": -49.23892593383789, | |
| "ref_logps/rejected": -32.732269287109375, | |
| "rewards/accuracies": 0.8671875, | |
| "rewards/chosen": -0.09847792983055115, | |
| "rewards/margins": 0.4775061011314392, | |
| "rewards/rejected": -0.575984001159668, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 24.210290197713302, | |
| "learning_rate": 4.017906269546778e-07, | |
| "logps/chosen": -48.78424072265625, | |
| "logps/rejected": -39.4119758605957, | |
| "loss": 0.5025, | |
| "losses/dpo": 0.2536649703979492, | |
| "losses/sft": 0.17507979273796082, | |
| "losses/total": 0.2536649703979492, | |
| "ref_logps/chosen": -47.147621154785156, | |
| "ref_logps/rejected": -32.35851287841797, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.16366226971149445, | |
| "rewards/margins": 0.5416839718818665, | |
| "rewards/rejected": -0.7053462266921997, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 25.054325101536794, | |
| "learning_rate": 3.920161866827889e-07, | |
| "logps/chosen": -46.48284912109375, | |
| "logps/rejected": -40.55732727050781, | |
| "loss": 0.5225, | |
| "losses/dpo": 0.6159500479698181, | |
| "losses/sft": 0.18471354246139526, | |
| "losses/total": 0.6159500479698181, | |
| "ref_logps/chosen": -44.64717102050781, | |
| "ref_logps/rejected": -34.08299255371094, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.18356791138648987, | |
| "rewards/margins": 0.46386560797691345, | |
| "rewards/rejected": -0.6474335193634033, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 25.059885652690767, | |
| "learning_rate": 3.8191006102653317e-07, | |
| "logps/chosen": -50.65240478515625, | |
| "logps/rejected": -44.85976028442383, | |
| "loss": 0.4509, | |
| "losses/dpo": 0.5429763793945312, | |
| "losses/sft": 0.19810011982917786, | |
| "losses/total": 0.5429763793945312, | |
| "ref_logps/chosen": -47.85638427734375, | |
| "ref_logps/rejected": -35.169281005859375, | |
| "rewards/accuracies": 0.9140625, | |
| "rewards/chosen": -0.27960240840911865, | |
| "rewards/margins": 0.6894451975822449, | |
| "rewards/rejected": -0.9690475463867188, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 19.99856582783424, | |
| "learning_rate": 3.7149585331065145e-07, | |
| "logps/chosen": -49.85383605957031, | |
| "logps/rejected": -45.81809997558594, | |
| "loss": 0.4332, | |
| "losses/dpo": 0.29431843757629395, | |
| "losses/sft": 0.18581561744213104, | |
| "losses/total": 0.29431843757629395, | |
| "ref_logps/chosen": -46.770938873291016, | |
| "ref_logps/rejected": -34.5809326171875, | |
| "rewards/accuracies": 0.8671875, | |
| "rewards/chosen": -0.3082895576953888, | |
| "rewards/margins": 0.8154268264770508, | |
| "rewards/rejected": -1.1237163543701172, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 34.79633257386577, | |
| "learning_rate": 3.6079788639981036e-07, | |
| "logps/chosen": -52.836326599121094, | |
| "logps/rejected": -46.93244934082031, | |
| "loss": 0.4604, | |
| "losses/dpo": 0.8810983300209045, | |
| "losses/sft": 0.23828193545341492, | |
| "losses/total": 0.8810983300209045, | |
| "ref_logps/chosen": -49.11648178100586, | |
| "ref_logps/rejected": -36.381752014160156, | |
| "rewards/accuracies": 0.8984375, | |
| "rewards/chosen": -0.3719848394393921, | |
| "rewards/margins": 0.6830847263336182, | |
| "rewards/rejected": -1.0550695657730103, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 23.026509844905394, | |
| "learning_rate": 3.498411458914238e-07, | |
| "logps/chosen": -50.38003921508789, | |
| "logps/rejected": -45.10429763793945, | |
| "loss": 0.4393, | |
| "losses/dpo": 0.15313033759593964, | |
| "losses/sft": 0.19763650000095367, | |
| "losses/total": 0.15313033759593964, | |
| "ref_logps/chosen": -46.028076171875, | |
| "ref_logps/rejected": -33.00657272338867, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.4351964592933655, | |
| "rewards/margins": 0.7745760679244995, | |
| "rewards/rejected": -1.2097725868225098, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 18.317574609447647, | |
| "learning_rate": 3.3865122176063385e-07, | |
| "logps/chosen": -51.4942512512207, | |
| "logps/rejected": -49.96583557128906, | |
| "loss": 0.4075, | |
| "losses/dpo": 0.1953999102115631, | |
| "losses/sft": 0.29790106415748596, | |
| "losses/total": 0.1953999102115631, | |
| "ref_logps/chosen": -45.6589469909668, | |
| "ref_logps/rejected": -34.858577728271484, | |
| "rewards/accuracies": 0.8515625, | |
| "rewards/chosen": -0.5835303068161011, | |
| "rewards/margins": 0.9271953105926514, | |
| "rewards/rejected": -1.510725736618042, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 19.255871137244554, | |
| "learning_rate": 3.272542485937368e-07, | |
| "logps/chosen": -50.351234436035156, | |
| "logps/rejected": -48.89935302734375, | |
| "loss": 0.3959, | |
| "losses/dpo": 0.4281933605670929, | |
| "losses/sft": 0.19774244725704193, | |
| "losses/total": 0.4281933605670929, | |
| "ref_logps/chosen": -43.48761749267578, | |
| "ref_logps/rejected": -32.255577087402344, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.68636155128479, | |
| "rewards/margins": 0.9780160188674927, | |
| "rewards/rejected": -1.6643775701522827, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 17.53385145494046, | |
| "learning_rate": 3.1567684454964674e-07, | |
| "logps/chosen": -49.46981430053711, | |
| "logps/rejected": -49.80710220336914, | |
| "loss": 0.4011, | |
| "losses/dpo": 0.5663512945175171, | |
| "losses/sft": 0.24904295802116394, | |
| "losses/total": 0.5663512945175171, | |
| "ref_logps/chosen": -42.88325500488281, | |
| "ref_logps/rejected": -33.13590621948242, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -0.6586559414863586, | |
| "rewards/margins": 1.0084636211395264, | |
| "rewards/rejected": -1.6671196222305298, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 157.5390863725062, | |
| "learning_rate": 3.0394604919195157e-07, | |
| "logps/chosen": -50.14772415161133, | |
| "logps/rejected": -49.97753143310547, | |
| "loss": 0.4132, | |
| "losses/dpo": 0.6134005784988403, | |
| "losses/sft": 0.1941785216331482, | |
| "losses/total": 0.6134005784988403, | |
| "ref_logps/chosen": -42.886375427246094, | |
| "ref_logps/rejected": -32.889442443847656, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.7261347770690918, | |
| "rewards/margins": 0.9826743006706238, | |
| "rewards/rejected": -1.7088091373443604, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 30.744138000924785, | |
| "learning_rate": 2.920892603367596e-07, | |
| "logps/chosen": -52.53690719604492, | |
| "logps/rejected": -51.7293701171875, | |
| "loss": 0.4345, | |
| "losses/dpo": 0.39982184767723083, | |
| "losses/sft": 0.16318069398403168, | |
| "losses/total": 0.39982184767723083, | |
| "ref_logps/chosen": -44.043270111083984, | |
| "ref_logps/rejected": -33.67184066772461, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": -0.8493636250495911, | |
| "rewards/margins": 0.956389307975769, | |
| "rewards/rejected": -1.8057528734207153, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 18.608606064784283, | |
| "learning_rate": 2.801341700638307e-07, | |
| "logps/chosen": -54.247406005859375, | |
| "logps/rejected": -51.46720886230469, | |
| "loss": 0.4308, | |
| "losses/dpo": 0.7559365630149841, | |
| "losses/sft": 0.20898960530757904, | |
| "losses/total": 0.7559365630149841, | |
| "ref_logps/chosen": -47.05962371826172, | |
| "ref_logps/rejected": -34.95857238769531, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.7187784910202026, | |
| "rewards/margins": 0.9320851564407349, | |
| "rewards/rejected": -1.6508636474609375, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 47.98397942977545, | |
| "learning_rate": 2.681087000404406e-07, | |
| "logps/chosen": -53.239768981933594, | |
| "logps/rejected": -52.34550476074219, | |
| "loss": 0.3907, | |
| "losses/dpo": 0.31572413444519043, | |
| "losses/sft": 0.18499067425727844, | |
| "losses/total": 0.31572413444519043, | |
| "ref_logps/chosen": -45.19135284423828, | |
| "ref_logps/rejected": -33.13307189941406, | |
| "rewards/accuracies": 0.90625, | |
| "rewards/chosen": -0.8048416972160339, | |
| "rewards/margins": 1.1164013147354126, | |
| "rewards/rejected": -1.9212429523468018, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 21.523748609052035, | |
| "learning_rate": 2.5604093630903305e-07, | |
| "logps/chosen": -53.806236267089844, | |
| "logps/rejected": -54.13373565673828, | |
| "loss": 0.3678, | |
| "losses/dpo": 0.6854045391082764, | |
| "losses/sft": 0.21097487211227417, | |
| "losses/total": 0.6854045391082764, | |
| "ref_logps/chosen": -44.96014404296875, | |
| "ref_logps/rejected": -34.04387664794922, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -0.8846092224121094, | |
| "rewards/margins": 1.1243770122528076, | |
| "rewards/rejected": -2.008985996246338, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 20.63046978113073, | |
| "learning_rate": 2.43959063690967e-07, | |
| "logps/chosen": -56.91130065917969, | |
| "logps/rejected": -54.714378356933594, | |
| "loss": 0.3872, | |
| "losses/dpo": 0.1204671785235405, | |
| "losses/sft": 0.17937365174293518, | |
| "losses/total": 0.1204671785235405, | |
| "ref_logps/chosen": -47.74310302734375, | |
| "ref_logps/rejected": -34.866615295410156, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.9168204069137573, | |
| "rewards/margins": 1.0679559707641602, | |
| "rewards/rejected": -1.984776258468628, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 27.841791874606287, | |
| "learning_rate": 2.3189129995955942e-07, | |
| "logps/chosen": -56.37548065185547, | |
| "logps/rejected": -55.140594482421875, | |
| "loss": 0.3703, | |
| "losses/dpo": 0.6694349646568298, | |
| "losses/sft": 0.15415219962596893, | |
| "losses/total": 0.6694349646568298, | |
| "ref_logps/chosen": -46.114707946777344, | |
| "ref_logps/rejected": -33.19464111328125, | |
| "rewards/accuracies": 0.8671875, | |
| "rewards/chosen": -1.0260775089263916, | |
| "rewards/margins": 1.16851806640625, | |
| "rewards/rejected": -2.1945955753326416, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 20.157417684445996, | |
| "learning_rate": 2.1986582993616925e-07, | |
| "logps/chosen": -55.861724853515625, | |
| "logps/rejected": -55.27591323852539, | |
| "loss": 0.4096, | |
| "losses/dpo": 0.253600537776947, | |
| "losses/sft": 0.25442296266555786, | |
| "losses/total": 0.253600537776947, | |
| "ref_logps/chosen": -46.024993896484375, | |
| "ref_logps/rejected": -34.88616180419922, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.9836731553077698, | |
| "rewards/margins": 1.0553019046783447, | |
| "rewards/rejected": -2.038975238800049, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 22.91868411351925, | |
| "learning_rate": 2.0791073966324034e-07, | |
| "logps/chosen": -56.3699836730957, | |
| "logps/rejected": -58.20032501220703, | |
| "loss": 0.3645, | |
| "losses/dpo": 0.05803808197379112, | |
| "losses/sft": 0.16261443495750427, | |
| "losses/total": 0.05803808197379112, | |
| "ref_logps/chosen": -46.18814468383789, | |
| "ref_logps/rejected": -35.7181396484375, | |
| "rewards/accuracies": 0.8828125, | |
| "rewards/chosen": -1.018183708190918, | |
| "rewards/margins": 1.230034351348877, | |
| "rewards/rejected": -2.248218059539795, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 24.665726952614282, | |
| "learning_rate": 1.960539508080485e-07, | |
| "logps/chosen": -55.33811569213867, | |
| "logps/rejected": -56.2475700378418, | |
| "loss": 0.4363, | |
| "losses/dpo": 0.6756047606468201, | |
| "losses/sft": 0.1989610195159912, | |
| "losses/total": 0.6756047606468201, | |
| "ref_logps/chosen": -42.876373291015625, | |
| "ref_logps/rejected": -33.306602478027344, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -1.2461739778518677, | |
| "rewards/margins": 1.0479230880737305, | |
| "rewards/rejected": -2.2940969467163086, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 35.04495782063734, | |
| "learning_rate": 1.8432315545035327e-07, | |
| "logps/chosen": -59.337791442871094, | |
| "logps/rejected": -60.82359313964844, | |
| "loss": 0.3701, | |
| "losses/dpo": 0.24237556755542755, | |
| "losses/sft": 0.14872561395168304, | |
| "losses/total": 0.24237556755542755, | |
| "ref_logps/chosen": -46.916419982910156, | |
| "ref_logps/rejected": -36.144935607910156, | |
| "rewards/accuracies": 0.8671875, | |
| "rewards/chosen": -1.2421373128890991, | |
| "rewards/margins": 1.2257287502288818, | |
| "rewards/rejected": -2.4678661823272705, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 18.874251761700755, | |
| "learning_rate": 1.7274575140626315e-07, | |
| "logps/chosen": -60.359886169433594, | |
| "logps/rejected": -56.043479919433594, | |
| "loss": 0.3903, | |
| "losses/dpo": 0.6876823902130127, | |
| "losses/sft": 0.163571298122406, | |
| "losses/total": 0.6876823902130127, | |
| "ref_logps/chosen": -49.23930358886719, | |
| "ref_logps/rejected": -34.02153778076172, | |
| "rewards/accuracies": 0.8984375, | |
| "rewards/chosen": -1.1120576858520508, | |
| "rewards/margins": 1.0901365280151367, | |
| "rewards/rejected": -2.2021942138671875, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 29.114539057876968, | |
| "learning_rate": 1.6134877823936607e-07, | |
| "logps/chosen": -60.98393249511719, | |
| "logps/rejected": -58.489444732666016, | |
| "loss": 0.4011, | |
| "losses/dpo": 0.03265048563480377, | |
| "losses/sft": 0.14689283072948456, | |
| "losses/total": 0.03265048563480377, | |
| "ref_logps/chosen": -49.34606170654297, | |
| "ref_logps/rejected": -36.67803955078125, | |
| "rewards/accuracies": 0.8671875, | |
| "rewards/chosen": -1.1637871265411377, | |
| "rewards/margins": 1.0173530578613281, | |
| "rewards/rejected": -2.181140184402466, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 21.107662898541907, | |
| "learning_rate": 1.5015885410857614e-07, | |
| "logps/chosen": -60.81307601928711, | |
| "logps/rejected": -59.90397262573242, | |
| "loss": 0.3897, | |
| "losses/dpo": 0.33075177669525146, | |
| "losses/sft": 0.214824840426445, | |
| "losses/total": 0.33075177669525146, | |
| "ref_logps/chosen": -46.25496292114258, | |
| "ref_logps/rejected": -33.91436004638672, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -1.4558112621307373, | |
| "rewards/margins": 1.143149971961975, | |
| "rewards/rejected": -2.598961114883423, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 26.95108201172052, | |
| "learning_rate": 1.392021136001897e-07, | |
| "logps/chosen": -56.23418426513672, | |
| "logps/rejected": -56.328125, | |
| "loss": 0.3964, | |
| "losses/dpo": 0.03794693946838379, | |
| "losses/sft": 0.19881302118301392, | |
| "losses/total": 0.03794693946838379, | |
| "ref_logps/chosen": -42.96794891357422, | |
| "ref_logps/rejected": -32.164451599121094, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.3266233205795288, | |
| "rewards/margins": 1.089743971824646, | |
| "rewards/rejected": -2.416367530822754, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 33.76828619344551, | |
| "learning_rate": 1.2850414668934847e-07, | |
| "logps/chosen": -61.50416946411133, | |
| "logps/rejected": -59.79325485229492, | |
| "loss": 0.3827, | |
| "losses/dpo": 0.5413109660148621, | |
| "losses/sft": 0.30467280745506287, | |
| "losses/total": 0.5413109660148621, | |
| "ref_logps/chosen": -48.96829605102539, | |
| "ref_logps/rejected": -35.99717330932617, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -1.2535876035690308, | |
| "rewards/margins": 1.1260210275650024, | |
| "rewards/rejected": -2.379608631134033, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 16.559964106722745, | |
| "learning_rate": 1.1808993897346678e-07, | |
| "logps/chosen": -58.611270904541016, | |
| "logps/rejected": -58.919395446777344, | |
| "loss": 0.3796, | |
| "losses/dpo": 0.3290981352329254, | |
| "losses/sft": 0.19547075033187866, | |
| "losses/total": 0.3290981352329254, | |
| "ref_logps/chosen": -46.96087646484375, | |
| "ref_logps/rejected": -36.086090087890625, | |
| "rewards/accuracies": 0.90625, | |
| "rewards/chosen": -1.1650400161743164, | |
| "rewards/margins": 1.1182900667190552, | |
| "rewards/rejected": -2.283329963684082, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 25.26391431571928, | |
| "learning_rate": 1.0798381331721107e-07, | |
| "logps/chosen": -58.2769775390625, | |
| "logps/rejected": -57.12656021118164, | |
| "loss": 0.3707, | |
| "losses/dpo": 0.3912191092967987, | |
| "losses/sft": 0.20826196670532227, | |
| "losses/total": 0.3912191092967987, | |
| "ref_logps/chosen": -46.01140213012695, | |
| "ref_logps/rejected": -32.54326629638672, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -1.226557731628418, | |
| "rewards/margins": 1.2317723035812378, | |
| "rewards/rejected": -2.4583301544189453, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 18.669814077600197, | |
| "learning_rate": 9.82093730453222e-08, | |
| "logps/chosen": -57.36506271362305, | |
| "logps/rejected": -57.83528137207031, | |
| "loss": 0.4249, | |
| "losses/dpo": 0.28024712204933167, | |
| "losses/sft": 0.21661897003650665, | |
| "losses/total": 0.28024712204933167, | |
| "ref_logps/chosen": -44.405941009521484, | |
| "ref_logps/rejected": -34.53661346435547, | |
| "rewards/accuracies": 0.8671875, | |
| "rewards/chosen": -1.295912265777588, | |
| "rewards/margins": 1.0339548587799072, | |
| "rewards/rejected": -2.329867124557495, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 17.65819121351904, | |
| "learning_rate": 8.87894468159574e-08, | |
| "logps/chosen": -60.354469299316406, | |
| "logps/rejected": -60.50645065307617, | |
| "loss": 0.3985, | |
| "losses/dpo": 0.9817911386489868, | |
| "losses/sft": 0.1904633343219757, | |
| "losses/total": 0.9817911386489868, | |
| "ref_logps/chosen": -46.499290466308594, | |
| "ref_logps/rejected": -34.763404846191406, | |
| "rewards/accuracies": 0.8359375, | |
| "rewards/chosen": -1.3855178356170654, | |
| "rewards/margins": 1.1887872219085693, | |
| "rewards/rejected": -2.5743050575256348, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 23.90292670438398, | |
| "learning_rate": 7.974603530330067e-08, | |
| "logps/chosen": -55.58333206176758, | |
| "logps/rejected": -55.52084732055664, | |
| "loss": 0.3777, | |
| "losses/dpo": 0.04075286537408829, | |
| "losses/sft": 0.22049269080162048, | |
| "losses/total": 0.04075286537408829, | |
| "ref_logps/chosen": -43.25560760498047, | |
| "ref_logps/rejected": -31.006759643554688, | |
| "rewards/accuracies": 0.8828125, | |
| "rewards/chosen": -1.2327725887298584, | |
| "rewards/margins": 1.2186365127563477, | |
| "rewards/rejected": -2.451408863067627, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 28.08593658686289, | |
| "learning_rate": 7.110025981396975e-08, | |
| "logps/chosen": -58.75514221191406, | |
| "logps/rejected": -58.784584045410156, | |
| "loss": 0.4449, | |
| "losses/dpo": 0.4793856143951416, | |
| "losses/sft": 0.20940393209457397, | |
| "losses/total": 0.4793856143951416, | |
| "ref_logps/chosen": -45.29600524902344, | |
| "ref_logps/rejected": -34.97162628173828, | |
| "rewards/accuracies": 0.8046875, | |
| "rewards/chosen": -1.3459134101867676, | |
| "rewards/margins": 1.0353822708129883, | |
| "rewards/rejected": -2.381295680999756, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 24.077339089176505, | |
| "learning_rate": 6.28723129572247e-08, | |
| "logps/chosen": -55.75697326660156, | |
| "logps/rejected": -56.72669219970703, | |
| "loss": 0.3567, | |
| "losses/dpo": 0.21238191425800323, | |
| "losses/sft": 0.1661817878484726, | |
| "losses/total": 0.21238191425800323, | |
| "ref_logps/chosen": -44.3855094909668, | |
| "ref_logps/rejected": -32.21479797363281, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -1.137147068977356, | |
| "rewards/margins": 1.314042568206787, | |
| "rewards/rejected": -2.4511895179748535, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 43.46612828134844, | |
| "learning_rate": 5.508141148419443e-08, | |
| "logps/chosen": -61.76049041748047, | |
| "logps/rejected": -62.041648864746094, | |
| "loss": 0.3688, | |
| "losses/dpo": 0.27996987104415894, | |
| "losses/sft": 0.1737639456987381, | |
| "losses/total": 0.27996987104415894, | |
| "ref_logps/chosen": -49.25553894042969, | |
| "ref_logps/rejected": -36.210182189941406, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.250495195388794, | |
| "rewards/margins": 1.3326513767242432, | |
| "rewards/rejected": -2.583146572113037, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 22.779198271573037, | |
| "learning_rate": 4.774575140626316e-08, | |
| "logps/chosen": -55.46681594848633, | |
| "logps/rejected": -57.17453384399414, | |
| "loss": 0.3531, | |
| "losses/dpo": 0.046613942831754684, | |
| "losses/sft": 0.20427729189395905, | |
| "losses/total": 0.046613942831754684, | |
| "ref_logps/chosen": -42.29081726074219, | |
| "ref_logps/rejected": -30.75497817993164, | |
| "rewards/accuracies": 0.8984375, | |
| "rewards/chosen": -1.3175995349884033, | |
| "rewards/margins": 1.3243558406829834, | |
| "rewards/rejected": -2.6419553756713867, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 20.59368424342303, | |
| "learning_rate": 4.0882465497443313e-08, | |
| "logps/chosen": -58.52223587036133, | |
| "logps/rejected": -56.04042053222656, | |
| "loss": 0.3923, | |
| "losses/dpo": 0.26003214716911316, | |
| "losses/sft": 0.17392012476921082, | |
| "losses/total": 0.26003214716911316, | |
| "ref_logps/chosen": -48.404632568359375, | |
| "ref_logps/rejected": -34.86602783203125, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -1.0117601156234741, | |
| "rewards/margins": 1.1056792736053467, | |
| "rewards/rejected": -2.1174392700195312, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 23.660376428219948, | |
| "learning_rate": 3.450758327998768e-08, | |
| "logps/chosen": -60.401039123535156, | |
| "logps/rejected": -60.10982131958008, | |
| "loss": 0.3902, | |
| "losses/dpo": 0.01773645170032978, | |
| "losses/sft": 0.17717282474040985, | |
| "losses/total": 0.01773645170032978, | |
| "ref_logps/chosen": -48.241943359375, | |
| "ref_logps/rejected": -34.582366943359375, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -1.215909719467163, | |
| "rewards/margins": 1.3368357419967651, | |
| "rewards/rejected": -2.5527453422546387, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 86.96881294099092, | |
| "learning_rate": 2.863599358669755e-08, | |
| "logps/chosen": -56.905418395996094, | |
| "logps/rejected": -56.808746337890625, | |
| "loss": 0.3944, | |
| "losses/dpo": 0.15065120160579681, | |
| "losses/sft": 0.22477349638938904, | |
| "losses/total": 0.15065120160579681, | |
| "ref_logps/chosen": -44.15583038330078, | |
| "ref_logps/rejected": -33.21840286254883, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.2749593257904053, | |
| "rewards/margins": 1.0840749740600586, | |
| "rewards/rejected": -2.359034299850464, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 18.8337077576639, | |
| "learning_rate": 2.3281409787363648e-08, | |
| "logps/chosen": -57.604774475097656, | |
| "logps/rejected": -57.78453063964844, | |
| "loss": 0.3863, | |
| "losses/dpo": 0.41682732105255127, | |
| "losses/sft": 0.16616390645503998, | |
| "losses/total": 0.41682732105255127, | |
| "ref_logps/chosen": -43.315818786621094, | |
| "ref_logps/rejected": -31.524248123168945, | |
| "rewards/accuracies": 0.8671875, | |
| "rewards/chosen": -1.4288955926895142, | |
| "rewards/margins": 1.1971325874328613, | |
| "rewards/rejected": -2.626028537750244, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 374.1054719017444, | |
| "learning_rate": 1.845633776055591e-08, | |
| "logps/chosen": -57.63691711425781, | |
| "logps/rejected": -58.3455810546875, | |
| "loss": 0.3882, | |
| "losses/dpo": 0.26508828997612, | |
| "losses/sft": 0.2718198001384735, | |
| "losses/total": 0.26508828997612, | |
| "ref_logps/chosen": -44.429481506347656, | |
| "ref_logps/rejected": -33.13744354248047, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.3207435607910156, | |
| "rewards/margins": 1.2000699043273926, | |
| "rewards/rejected": -2.520813465118408, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 26.70970124014032, | |
| "learning_rate": 1.4172046685564209e-08, | |
| "logps/chosen": -58.663551330566406, | |
| "logps/rejected": -58.07282257080078, | |
| "loss": 0.3962, | |
| "losses/dpo": 0.08177483081817627, | |
| "losses/sft": 0.18531636893749237, | |
| "losses/total": 0.08177483081817627, | |
| "ref_logps/chosen": -45.821983337402344, | |
| "ref_logps/rejected": -33.62261199951172, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.2841567993164062, | |
| "rewards/margins": 1.1608643531799316, | |
| "rewards/rejected": -2.445021390914917, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 25.593261462625442, | |
| "learning_rate": 1.0438542722708444e-08, | |
| "logps/chosen": -59.08097839355469, | |
| "logps/rejected": -59.16502380371094, | |
| "loss": 0.3836, | |
| "losses/dpo": 0.02788337506353855, | |
| "losses/sft": 0.19819076359272003, | |
| "losses/total": 0.02788337506353855, | |
| "ref_logps/chosen": -45.94892883300781, | |
| "ref_logps/rejected": -33.597511291503906, | |
| "rewards/accuracies": 0.8828125, | |
| "rewards/chosen": -1.3132052421569824, | |
| "rewards/margins": 1.2435462474822998, | |
| "rewards/rejected": -2.556751251220703, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 25.28796063034412, | |
| "learning_rate": 7.2645456434869965e-09, | |
| "logps/chosen": -57.95222473144531, | |
| "logps/rejected": -58.91720199584961, | |
| "loss": 0.3915, | |
| "losses/dpo": 1.2907841205596924, | |
| "losses/sft": 0.20458956062793732, | |
| "losses/total": 1.2907841205596924, | |
| "ref_logps/chosen": -45.50114440917969, | |
| "ref_logps/rejected": -35.063446044921875, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -1.2451080083847046, | |
| "rewards/margins": 1.140267252922058, | |
| "rewards/rejected": -2.385375499725342, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 30.554099185463503, | |
| "learning_rate": 4.657468465146641e-09, | |
| "logps/chosen": -57.99516296386719, | |
| "logps/rejected": -55.496768951416016, | |
| "loss": 0.3752, | |
| "losses/dpo": 0.20264464616775513, | |
| "losses/sft": 0.17493540048599243, | |
| "losses/total": 0.20264464616775513, | |
| "ref_logps/chosen": -47.58026123046875, | |
| "ref_logps/rejected": -33.345062255859375, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -1.041489839553833, | |
| "rewards/margins": 1.1736811399459839, | |
| "rewards/rejected": -2.2151710987091064, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 21.555895701368716, | |
| "learning_rate": 2.6234001372372193e-09, | |
| "logps/chosen": -55.79784393310547, | |
| "logps/rejected": -54.85697555541992, | |
| "loss": 0.4513, | |
| "losses/dpo": 0.6288288235664368, | |
| "losses/sft": 0.25858786702156067, | |
| "losses/total": 0.6288288235664368, | |
| "ref_logps/chosen": -42.008121490478516, | |
| "ref_logps/rejected": -31.47281265258789, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.3789721727371216, | |
| "rewards/margins": 0.9594441056251526, | |
| "rewards/rejected": -2.338416337966919, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 21.73384383499147, | |
| "learning_rate": 1.167091320587843e-09, | |
| "logps/chosen": -56.99696350097656, | |
| "logps/rejected": -59.2013053894043, | |
| "loss": 0.3554, | |
| "losses/dpo": 0.09169570356607437, | |
| "losses/sft": 0.20991858839988708, | |
| "losses/total": 0.09169570356607437, | |
| "ref_logps/chosen": -42.36278533935547, | |
| "ref_logps/rejected": -31.79424476623535, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -1.463417887687683, | |
| "rewards/margins": 1.2772881984710693, | |
| "rewards/rejected": -2.740705966949463, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 30.958564799186906, | |
| "learning_rate": 2.9194329191833953e-10, | |
| "logps/chosen": -58.35291290283203, | |
| "logps/rejected": -56.74859619140625, | |
| "loss": 0.3706, | |
| "losses/dpo": 0.3077165484428406, | |
| "losses/sft": 0.17356029152870178, | |
| "losses/total": 0.3077165484428406, | |
| "ref_logps/chosen": -44.90869903564453, | |
| "ref_logps/rejected": -31.324697494506836, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -1.34442138671875, | |
| "rewards/margins": 1.197968602180481, | |
| "rewards/rejected": -2.5423898696899414, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 20.514487251091158, | |
| "learning_rate": 0.0, | |
| "logps/chosen": -55.3281135559082, | |
| "logps/rejected": -54.42873764038086, | |
| "loss": 0.4185, | |
| "losses/dpo": 0.45331382751464844, | |
| "losses/sft": 0.16170088946819305, | |
| "losses/total": 0.45331382751464844, | |
| "ref_logps/chosen": -42.832916259765625, | |
| "ref_logps/rejected": -31.545093536376953, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.2495195865631104, | |
| "rewards/margins": 1.0388449430465698, | |
| "rewards/rejected": -2.2883644104003906, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 73, | |
| "total_flos": 0.0, | |
| "train_loss": 0.4880054197082781, | |
| "train_runtime": 1195.1879, | |
| "train_samples_per_second": 7.883, | |
| "train_steps_per_second": 0.061 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 73, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 1000, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |