| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.6615384615384614, |
| "eval_steps": 20, |
| "global_step": 360, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.009230769230769232, |
| "grad_norm": 52.40730345789634, |
| "learning_rate": 2.2727272727272725e-08, |
| "logits/chosen": -1.2901445627212524, |
| "logits/rejected": -1.2963205575942993, |
| "logps/chosen": -16.113027572631836, |
| "logps/rejected": -27.10122299194336, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.018461538461538463, |
| "grad_norm": 64.88802449206628, |
| "learning_rate": 4.545454545454545e-08, |
| "logits/chosen": -1.3016295433044434, |
| "logits/rejected": -1.3255655765533447, |
| "logps/chosen": -20.355079650878906, |
| "logps/rejected": -39.93232727050781, |
| "loss": 0.6895, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.008816350251436234, |
| "rewards/margins": 0.0047285472974181175, |
| "rewards/rejected": 0.004087802488356829, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.027692307692307693, |
| "grad_norm": 59.6800701771534, |
| "learning_rate": 6.818181818181817e-08, |
| "logits/chosen": -1.31508207321167, |
| "logits/rejected": -1.3189733028411865, |
| "logps/chosen": -23.069622039794922, |
| "logps/rejected": -26.97477149963379, |
| "loss": 0.695, |
| "rewards/accuracies": 0.4444444477558136, |
| "rewards/chosen": -0.007372706197202206, |
| "rewards/margins": -0.013017671182751656, |
| "rewards/rejected": 0.0056449659168720245, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.036923076923076927, |
| "grad_norm": 52.983511533208585, |
| "learning_rate": 9.09090909090909e-08, |
| "logits/chosen": -1.277503252029419, |
| "logits/rejected": -1.3002785444259644, |
| "logps/chosen": -20.34660530090332, |
| "logps/rejected": -31.0557861328125, |
| "loss": 0.6908, |
| "rewards/accuracies": 0.4861111044883728, |
| "rewards/chosen": 0.020842621102929115, |
| "rewards/margins": 0.020597590133547783, |
| "rewards/rejected": 0.00024503222084604204, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.046153846153846156, |
| "grad_norm": 66.2747581823961, |
| "learning_rate": 1.1363636363636363e-07, |
| "logits/chosen": -1.3306350708007812, |
| "logits/rejected": -1.3309379816055298, |
| "logps/chosen": -26.48358917236328, |
| "logps/rejected": -30.445173263549805, |
| "loss": 0.7046, |
| "rewards/accuracies": 0.3888888955116272, |
| "rewards/chosen": -0.01246996782720089, |
| "rewards/margins": -0.032543592154979706, |
| "rewards/rejected": 0.020073626190423965, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.055384615384615386, |
| "grad_norm": 57.271529486531605, |
| "learning_rate": 1.3636363636363635e-07, |
| "logits/chosen": -1.280084252357483, |
| "logits/rejected": -1.295721411705017, |
| "logps/chosen": -25.79343032836914, |
| "logps/rejected": -36.58183288574219, |
| "loss": 0.6956, |
| "rewards/accuracies": 0.5555555820465088, |
| "rewards/chosen": 0.023966560140252113, |
| "rewards/margins": 0.030559096485376358, |
| "rewards/rejected": -0.006592527963221073, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.06461538461538462, |
| "grad_norm": 67.94854888195144, |
| "learning_rate": 1.5909090909090907e-07, |
| "logits/chosen": -1.2790985107421875, |
| "logits/rejected": -1.296931266784668, |
| "logps/chosen": -24.833446502685547, |
| "logps/rejected": -31.11182403564453, |
| "loss": 0.7006, |
| "rewards/accuracies": 0.4305555522441864, |
| "rewards/chosen": 0.013436201959848404, |
| "rewards/margins": 0.002752000233158469, |
| "rewards/rejected": 0.010684202425181866, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.07384615384615385, |
| "grad_norm": 49.36191286721225, |
| "learning_rate": 1.818181818181818e-07, |
| "logits/chosen": -1.301368236541748, |
| "logits/rejected": -1.3136367797851562, |
| "logps/chosen": -26.273963928222656, |
| "logps/rejected": -35.63306427001953, |
| "loss": 0.6949, |
| "rewards/accuracies": 0.5972222089767456, |
| "rewards/chosen": 0.015296169556677341, |
| "rewards/margins": 0.013788570649921894, |
| "rewards/rejected": 0.0015075993724167347, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.08307692307692308, |
| "grad_norm": 56.43976674406361, |
| "learning_rate": 2.0454545454545456e-07, |
| "logits/chosen": -1.3201720714569092, |
| "logits/rejected": -1.3183202743530273, |
| "logps/chosen": -25.70770263671875, |
| "logps/rejected": -26.178009033203125, |
| "loss": 0.7006, |
| "rewards/accuracies": 0.5555555820465088, |
| "rewards/chosen": 0.0011544560547918081, |
| "rewards/margins": 0.01863468438386917, |
| "rewards/rejected": -0.01748022995889187, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.09230769230769231, |
| "grad_norm": 56.010590202518365, |
| "learning_rate": 2.2727272727272726e-07, |
| "logits/chosen": -1.2482044696807861, |
| "logits/rejected": -1.262031078338623, |
| "logps/chosen": -28.337791442871094, |
| "logps/rejected": -29.38203239440918, |
| "loss": 0.6883, |
| "rewards/accuracies": 0.4861111044883728, |
| "rewards/chosen": 0.00024333276087418199, |
| "rewards/margins": -0.0005785864195786417, |
| "rewards/rejected": 0.000821918249130249, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.09230769230769231, |
| "eval_logits/chosen": -1.3220677375793457, |
| "eval_logits/rejected": -1.33245849609375, |
| "eval_logps/chosen": -23.036666870117188, |
| "eval_logps/rejected": -26.372356414794922, |
| "eval_loss": 0.6916412115097046, |
| "eval_rewards/accuracies": 0.4965437650680542, |
| "eval_rewards/chosen": 0.00501647312194109, |
| "eval_rewards/margins": 0.010797887109220028, |
| "eval_rewards/rejected": -0.0057814153842628, |
| "eval_runtime": 216.2201, |
| "eval_samples_per_second": 8.02, |
| "eval_steps_per_second": 2.007, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.10153846153846154, |
| "grad_norm": 67.30805212172523, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -1.2273086309432983, |
| "logits/rejected": -1.2565299272537231, |
| "logps/chosen": -21.540626525878906, |
| "logps/rejected": -47.4769172668457, |
| "loss": 0.6893, |
| "rewards/accuracies": 0.4861111044883728, |
| "rewards/chosen": 0.007773838937282562, |
| "rewards/margins": 0.026619136333465576, |
| "rewards/rejected": -0.018845297396183014, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.11076923076923077, |
| "grad_norm": 51.29780655120263, |
| "learning_rate": 2.727272727272727e-07, |
| "logits/chosen": -1.219795823097229, |
| "logits/rejected": -1.235877513885498, |
| "logps/chosen": -30.82242774963379, |
| "logps/rejected": -37.68511962890625, |
| "loss": 0.6758, |
| "rewards/accuracies": 0.5277777910232544, |
| "rewards/chosen": 0.03086034394800663, |
| "rewards/margins": 0.055920813232660294, |
| "rewards/rejected": -0.025060458108782768, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.12, |
| "grad_norm": 55.0939959360046, |
| "learning_rate": 2.9545454545454545e-07, |
| "logits/chosen": -1.258486270904541, |
| "logits/rejected": -1.2752680778503418, |
| "logps/chosen": -25.136966705322266, |
| "logps/rejected": -43.23137664794922, |
| "loss": 0.6774, |
| "rewards/accuracies": 0.5972222089767456, |
| "rewards/chosen": 0.0216163769364357, |
| "rewards/margins": 0.08480846881866455, |
| "rewards/rejected": -0.06319208443164825, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.12923076923076923, |
| "grad_norm": 48.332663649143974, |
| "learning_rate": 3.1818181818181815e-07, |
| "logits/chosen": -1.320160150527954, |
| "logits/rejected": -1.330212950706482, |
| "logps/chosen": -19.24217414855957, |
| "logps/rejected": -27.22931671142578, |
| "loss": 0.6874, |
| "rewards/accuracies": 0.4027777910232544, |
| "rewards/chosen": 0.01321298535913229, |
| "rewards/margins": 0.009595979005098343, |
| "rewards/rejected": 0.0036170051898807287, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.13846153846153847, |
| "grad_norm": 49.59877928678631, |
| "learning_rate": 3.4090909090909085e-07, |
| "logits/chosen": -1.2795339822769165, |
| "logits/rejected": -1.2929219007492065, |
| "logps/chosen": -21.841049194335938, |
| "logps/rejected": -28.89714813232422, |
| "loss": 0.6813, |
| "rewards/accuracies": 0.5555555820465088, |
| "rewards/chosen": 0.013576723635196686, |
| "rewards/margins": 0.05021868646144867, |
| "rewards/rejected": -0.036641962826251984, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.1476923076923077, |
| "grad_norm": 51.02397460357053, |
| "learning_rate": 3.636363636363636e-07, |
| "logits/chosen": -1.2797447443008423, |
| "logits/rejected": -1.3022751808166504, |
| "logps/chosen": -24.65501594543457, |
| "logps/rejected": -36.741573333740234, |
| "loss": 0.6732, |
| "rewards/accuracies": 0.6527777910232544, |
| "rewards/chosen": 0.04290567338466644, |
| "rewards/margins": 0.09170582890510559, |
| "rewards/rejected": -0.04880015552043915, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.15692307692307692, |
| "grad_norm": 45.39524675384609, |
| "learning_rate": 3.8636363636363636e-07, |
| "logits/chosen": -1.2498574256896973, |
| "logits/rejected": -1.2657580375671387, |
| "logps/chosen": -21.32640838623047, |
| "logps/rejected": -39.71310806274414, |
| "loss": 0.6627, |
| "rewards/accuracies": 0.5277777910232544, |
| "rewards/chosen": 0.007356289308518171, |
| "rewards/margins": 0.06605351716279984, |
| "rewards/rejected": -0.058697231113910675, |
| "step": 34 |
| }, |
| { |
| "epoch": 0.16615384615384615, |
| "grad_norm": 52.58099443727954, |
| "learning_rate": 4.090909090909091e-07, |
| "logits/chosen": -1.2139866352081299, |
| "logits/rejected": -1.2340948581695557, |
| "logps/chosen": -18.409015655517578, |
| "logps/rejected": -35.20015335083008, |
| "loss": 0.6644, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.022290384396910667, |
| "rewards/margins": 0.06140115484595299, |
| "rewards/rejected": -0.03911077231168747, |
| "step": 36 |
| }, |
| { |
| "epoch": 0.1753846153846154, |
| "grad_norm": 53.938952453151614, |
| "learning_rate": 4.318181818181818e-07, |
| "logits/chosen": -1.2461514472961426, |
| "logits/rejected": -1.2598522901535034, |
| "logps/chosen": -27.248275756835938, |
| "logps/rejected": -32.50380325317383, |
| "loss": 0.6545, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.04994047060608864, |
| "rewards/margins": 0.1001262366771698, |
| "rewards/rejected": -0.05018576979637146, |
| "step": 38 |
| }, |
| { |
| "epoch": 0.18461538461538463, |
| "grad_norm": 46.949545804629956, |
| "learning_rate": 4.545454545454545e-07, |
| "logits/chosen": -1.2425076961517334, |
| "logits/rejected": -1.2611976861953735, |
| "logps/chosen": -14.459053993225098, |
| "logps/rejected": -22.981327056884766, |
| "loss": 0.6562, |
| "rewards/accuracies": 0.5833333134651184, |
| "rewards/chosen": 0.06477613002061844, |
| "rewards/margins": 0.08449113368988037, |
| "rewards/rejected": -0.019715001806616783, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.18461538461538463, |
| "eval_logits/chosen": -1.3191018104553223, |
| "eval_logits/rejected": -1.3294612169265747, |
| "eval_logps/chosen": -22.93289566040039, |
| "eval_logps/rejected": -26.52239418029785, |
| "eval_loss": 0.6399217247962952, |
| "eval_rewards/accuracies": 0.671658992767334, |
| "eval_rewards/chosen": 0.05690104886889458, |
| "eval_rewards/margins": 0.13770265877246857, |
| "eval_rewards/rejected": -0.08080162853002548, |
| "eval_runtime": 216.334, |
| "eval_samples_per_second": 8.015, |
| "eval_steps_per_second": 2.006, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.19384615384615383, |
| "grad_norm": 41.53559188167412, |
| "learning_rate": 4.772727272727273e-07, |
| "logits/chosen": -1.2119545936584473, |
| "logits/rejected": -1.2175490856170654, |
| "logps/chosen": -23.42240333557129, |
| "logps/rejected": -29.862327575683594, |
| "loss": 0.624, |
| "rewards/accuracies": 0.5694444179534912, |
| "rewards/chosen": 0.059619419276714325, |
| "rewards/margins": 0.15751110017299652, |
| "rewards/rejected": -0.09789170324802399, |
| "step": 42 |
| }, |
| { |
| "epoch": 0.20307692307692307, |
| "grad_norm": 49.942474151893265, |
| "learning_rate": 5e-07, |
| "logits/chosen": -1.3206286430358887, |
| "logits/rejected": -1.3300279378890991, |
| "logps/chosen": -22.983713150024414, |
| "logps/rejected": -23.000356674194336, |
| "loss": 0.6224, |
| "rewards/accuracies": 0.7083333134651184, |
| "rewards/chosen": 0.05540511757135391, |
| "rewards/margins": 0.1078185960650444, |
| "rewards/rejected": -0.05241347849369049, |
| "step": 44 |
| }, |
| { |
| "epoch": 0.2123076923076923, |
| "grad_norm": 40.96104792630147, |
| "learning_rate": 4.99967220916408e-07, |
| "logits/chosen": -1.2594552040100098, |
| "logits/rejected": -1.270306944847107, |
| "logps/chosen": -19.131641387939453, |
| "logps/rejected": -29.00514793395996, |
| "loss": 0.617, |
| "rewards/accuracies": 0.7083333134651184, |
| "rewards/chosen": 0.09214716404676437, |
| "rewards/margins": 0.24131464958190918, |
| "rewards/rejected": -0.14916746318340302, |
| "step": 46 |
| }, |
| { |
| "epoch": 0.22153846153846155, |
| "grad_norm": 44.60792696333844, |
| "learning_rate": 4.998688922613787e-07, |
| "logits/chosen": -1.3020961284637451, |
| "logits/rejected": -1.3101927042007446, |
| "logps/chosen": -31.274911880493164, |
| "logps/rejected": -32.11240005493164, |
| "loss": 0.6075, |
| "rewards/accuracies": 0.7083333134651184, |
| "rewards/chosen": 0.10760927200317383, |
| "rewards/margins": 0.294413298368454, |
| "rewards/rejected": -0.18680399656295776, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.23076923076923078, |
| "grad_norm": 43.17860095734465, |
| "learning_rate": 4.997050398198976e-07, |
| "logits/chosen": -1.291076421737671, |
| "logits/rejected": -1.2982360124588013, |
| "logps/chosen": -22.59940528869629, |
| "logps/rejected": -22.504961013793945, |
| "loss": 0.5855, |
| "rewards/accuracies": 0.7638888955116272, |
| "rewards/chosen": 0.14835722744464874, |
| "rewards/margins": 0.3006143271923065, |
| "rewards/rejected": -0.15225709974765778, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.24, |
| "grad_norm": 40.923959372883246, |
| "learning_rate": 4.994757065594279e-07, |
| "logits/chosen": -1.2361193895339966, |
| "logits/rejected": -1.2530244588851929, |
| "logps/chosen": -19.440345764160156, |
| "logps/rejected": -29.653764724731445, |
| "loss": 0.58, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.12961499392986298, |
| "rewards/margins": 0.2747644782066345, |
| "rewards/rejected": -0.14514949917793274, |
| "step": 52 |
| }, |
| { |
| "epoch": 0.24923076923076923, |
| "grad_norm": 40.97149688332116, |
| "learning_rate": 4.991809526186423e-07, |
| "logits/chosen": -1.2297606468200684, |
| "logits/rejected": -1.25152587890625, |
| "logps/chosen": -21.388309478759766, |
| "logps/rejected": -44.34809112548828, |
| "loss": 0.5456, |
| "rewards/accuracies": 0.7777777910232544, |
| "rewards/chosen": 0.151195228099823, |
| "rewards/margins": 0.48822492361068726, |
| "rewards/rejected": -0.33702969551086426, |
| "step": 54 |
| }, |
| { |
| "epoch": 0.25846153846153846, |
| "grad_norm": 41.37645783028047, |
| "learning_rate": 4.988208552916535e-07, |
| "logits/chosen": -1.2540967464447021, |
| "logits/rejected": -1.2566981315612793, |
| "logps/chosen": -22.95637321472168, |
| "logps/rejected": -23.91745376586914, |
| "loss": 0.5722, |
| "rewards/accuracies": 0.7916666865348816, |
| "rewards/chosen": 0.20567570626735687, |
| "rewards/margins": 0.3446711003780365, |
| "rewards/rejected": -0.13899540901184082, |
| "step": 56 |
| }, |
| { |
| "epoch": 0.2676923076923077, |
| "grad_norm": 37.07709893155658, |
| "learning_rate": 4.983955090077444e-07, |
| "logits/chosen": -1.2924391031265259, |
| "logits/rejected": -1.2913458347320557, |
| "logps/chosen": -18.923715591430664, |
| "logps/rejected": -22.57257843017578, |
| "loss": 0.5773, |
| "rewards/accuracies": 0.7916666865348816, |
| "rewards/chosen": 0.16216707229614258, |
| "rewards/margins": 0.27626025676727295, |
| "rewards/rejected": -0.11409316956996918, |
| "step": 58 |
| }, |
| { |
| "epoch": 0.27692307692307694, |
| "grad_norm": 33.00415567764037, |
| "learning_rate": 4.979050253066063e-07, |
| "logits/chosen": -1.2263813018798828, |
| "logits/rejected": -1.2465788125991821, |
| "logps/chosen": -20.503381729125977, |
| "logps/rejected": -37.98419189453125, |
| "loss": 0.5379, |
| "rewards/accuracies": 0.7083333134651184, |
| "rewards/chosen": 0.17731823027133942, |
| "rewards/margins": 0.593184769153595, |
| "rewards/rejected": -0.41586652398109436, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.27692307692307694, |
| "eval_logits/chosen": -1.303908109664917, |
| "eval_logits/rejected": -1.3140496015548706, |
| "eval_logps/chosen": -22.596784591674805, |
| "eval_logps/rejected": -26.880229949951172, |
| "eval_loss": 0.5301286578178406, |
| "eval_rewards/accuracies": 0.7718893885612488, |
| "eval_rewards/chosen": 0.22495588660240173, |
| "eval_rewards/margins": 0.484672486782074, |
| "eval_rewards/rejected": -0.259716659784317, |
| "eval_runtime": 215.7229, |
| "eval_samples_per_second": 8.038, |
| "eval_steps_per_second": 2.012, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.28615384615384615, |
| "grad_norm": 32.870504270075905, |
| "learning_rate": 4.973495328090889e-07, |
| "logits/chosen": -1.2028117179870605, |
| "logits/rejected": -1.2163152694702148, |
| "logps/chosen": -25.100025177001953, |
| "logps/rejected": -35.97075653076172, |
| "loss": 0.5245, |
| "rewards/accuracies": 0.7083333134651184, |
| "rewards/chosen": 0.20213226974010468, |
| "rewards/margins": 0.5411441326141357, |
| "rewards/rejected": -0.33901187777519226, |
| "step": 62 |
| }, |
| { |
| "epoch": 0.2953846153846154, |
| "grad_norm": 38.13033333375434, |
| "learning_rate": 4.967291771834726e-07, |
| "logits/chosen": -1.2682946920394897, |
| "logits/rejected": -1.2830837965011597, |
| "logps/chosen": -22.399858474731445, |
| "logps/rejected": -35.47315979003906, |
| "loss": 0.4854, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.24411238729953766, |
| "rewards/margins": 0.7097706198692322, |
| "rewards/rejected": -0.46565818786621094, |
| "step": 64 |
| }, |
| { |
| "epoch": 0.3046153846153846, |
| "grad_norm": 34.6917991893696, |
| "learning_rate": 4.960441211072685e-07, |
| "logits/chosen": -1.240267038345337, |
| "logits/rejected": -1.2494441270828247, |
| "logps/chosen": -16.752328872680664, |
| "logps/rejected": -21.625200271606445, |
| "loss": 0.52, |
| "rewards/accuracies": 0.7916666865348816, |
| "rewards/chosen": 0.2749379575252533, |
| "rewards/margins": 0.5106962323188782, |
| "rewards/rejected": -0.23575833439826965, |
| "step": 66 |
| }, |
| { |
| "epoch": 0.31384615384615383, |
| "grad_norm": 32.938257449212315, |
| "learning_rate": 4.952945442245597e-07, |
| "logits/chosen": -1.282260775566101, |
| "logits/rejected": -1.2961454391479492, |
| "logps/chosen": -16.818540573120117, |
| "logps/rejected": -31.804317474365234, |
| "loss": 0.4986, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.20085500180721283, |
| "rewards/margins": 0.6287386417388916, |
| "rewards/rejected": -0.42788365483283997, |
| "step": 68 |
| }, |
| { |
| "epoch": 0.3230769230769231, |
| "grad_norm": 36.12880857430109, |
| "learning_rate": 4.944806430988927e-07, |
| "logits/chosen": -1.2567392587661743, |
| "logits/rejected": -1.263179063796997, |
| "logps/chosen": -23.333267211914062, |
| "logps/rejected": -24.862985610961914, |
| "loss": 0.5059, |
| "rewards/accuracies": 0.7361111044883728, |
| "rewards/chosen": 0.22914116084575653, |
| "rewards/margins": 0.6000176668167114, |
| "rewards/rejected": -0.3708764612674713, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.3323076923076923, |
| "grad_norm": 31.746333807337315, |
| "learning_rate": 4.936026311617316e-07, |
| "logits/chosen": -1.2413491010665894, |
| "logits/rejected": -1.2490180730819702, |
| "logps/chosen": -27.870990753173828, |
| "logps/rejected": -28.86038589477539, |
| "loss": 0.4797, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.35419517755508423, |
| "rewards/margins": 0.7417442202568054, |
| "rewards/rejected": -0.3875490427017212, |
| "step": 72 |
| }, |
| { |
| "epoch": 0.3415384615384615, |
| "grad_norm": 31.965936446320438, |
| "learning_rate": 4.926607386564898e-07, |
| "logits/chosen": -1.3071357011795044, |
| "logits/rejected": -1.3031624555587769, |
| "logps/chosen": -24.66501808166504, |
| "logps/rejected": -19.646629333496094, |
| "loss": 0.4724, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.3141394257545471, |
| "rewards/margins": 0.6052231788635254, |
| "rewards/rejected": -0.29108375310897827, |
| "step": 74 |
| }, |
| { |
| "epoch": 0.3507692307692308, |
| "grad_norm": 30.420218056003396, |
| "learning_rate": 4.916552125781528e-07, |
| "logits/chosen": -1.2826448678970337, |
| "logits/rejected": -1.2921828031539917, |
| "logps/chosen": -21.71385955810547, |
| "logps/rejected": -26.265592575073242, |
| "loss": 0.443, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.3491870164871216, |
| "rewards/margins": 0.7558759450912476, |
| "rewards/rejected": -0.4066888988018036, |
| "step": 76 |
| }, |
| { |
| "epoch": 0.36, |
| "grad_norm": 35.262762131347294, |
| "learning_rate": 4.905863166085075e-07, |
| "logits/chosen": -1.2882230281829834, |
| "logits/rejected": -1.3004416227340698, |
| "logps/chosen": -25.61620330810547, |
| "logps/rejected": -26.73788833618164, |
| "loss": 0.4682, |
| "rewards/accuracies": 0.7638888955116272, |
| "rewards/chosen": 0.29705706238746643, |
| "rewards/margins": 0.6734262108802795, |
| "rewards/rejected": -0.3763691484928131, |
| "step": 78 |
| }, |
| { |
| "epoch": 0.36923076923076925, |
| "grad_norm": 32.454214562336674, |
| "learning_rate": 4.894543310469967e-07, |
| "logits/chosen": -1.292490839958191, |
| "logits/rejected": -1.3075741529464722, |
| "logps/chosen": -24.23374366760254, |
| "logps/rejected": -27.662269592285156, |
| "loss": 0.4233, |
| "rewards/accuracies": 0.7222222089767456, |
| "rewards/chosen": 0.3347330093383789, |
| "rewards/margins": 0.7462683320045471, |
| "rewards/rejected": -0.4115353524684906, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.36923076923076925, |
| "eval_logits/chosen": -1.2837809324264526, |
| "eval_logits/rejected": -1.293448567390442, |
| "eval_logps/chosen": -22.318069458007812, |
| "eval_logps/rejected": -27.420156478881836, |
| "eval_loss": 0.4364205598831177, |
| "eval_rewards/accuracies": 0.7937787771224976, |
| "eval_rewards/chosen": 0.3643138110637665, |
| "eval_rewards/margins": 0.893993616104126, |
| "eval_rewards/rejected": -0.5296797752380371, |
| "eval_runtime": 215.7088, |
| "eval_samples_per_second": 8.039, |
| "eval_steps_per_second": 2.012, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.37846153846153846, |
| "grad_norm": 30.100728508551764, |
| "learning_rate": 4.882595527372152e-07, |
| "logits/chosen": -1.219198226928711, |
| "logits/rejected": -1.2316464185714722, |
| "logps/chosen": -21.758522033691406, |
| "logps/rejected": -32.21995544433594, |
| "loss": 0.4544, |
| "rewards/accuracies": 0.7638888955116272, |
| "rewards/chosen": 0.33725497126579285, |
| "rewards/margins": 0.9134353995323181, |
| "rewards/rejected": -0.5761803984642029, |
| "step": 82 |
| }, |
| { |
| "epoch": 0.38769230769230767, |
| "grad_norm": 27.99260854977849, |
| "learning_rate": 4.870022949890676e-07, |
| "logits/chosen": -1.25475013256073, |
| "logits/rejected": -1.258756160736084, |
| "logps/chosen": -29.569332122802734, |
| "logps/rejected": -32.13206481933594, |
| "loss": 0.4048, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.3496508300304413, |
| "rewards/margins": 1.0080742835998535, |
| "rewards/rejected": -0.6584234237670898, |
| "step": 84 |
| }, |
| { |
| "epoch": 0.39692307692307693, |
| "grad_norm": 28.434505768144174, |
| "learning_rate": 4.856828874966086e-07, |
| "logits/chosen": -1.2163680791854858, |
| "logits/rejected": -1.2340407371520996, |
| "logps/chosen": -18.534114837646484, |
| "logps/rejected": -36.619850158691406, |
| "loss": 0.422, |
| "rewards/accuracies": 0.6666666865348816, |
| "rewards/chosen": 0.2995716333389282, |
| "rewards/margins": 1.0983738899230957, |
| "rewards/rejected": -0.7988021969795227, |
| "step": 86 |
| }, |
| { |
| "epoch": 0.40615384615384614, |
| "grad_norm": 28.794469436567187, |
| "learning_rate": 4.843016762515859e-07, |
| "logits/chosen": -1.2752939462661743, |
| "logits/rejected": -1.285552978515625, |
| "logps/chosen": -21.55384635925293, |
| "logps/rejected": -30.397226333618164, |
| "loss": 0.3905, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.37557560205459595, |
| "rewards/margins": 1.0376694202423096, |
| "rewards/rejected": -0.6620937585830688, |
| "step": 88 |
| }, |
| { |
| "epoch": 0.4153846153846154, |
| "grad_norm": 24.699190483704957, |
| "learning_rate": 4.828590234527106e-07, |
| "logits/chosen": -1.2076385021209717, |
| "logits/rejected": -1.2378058433532715, |
| "logps/chosen": -20.13502311706543, |
| "logps/rejected": -49.50822067260742, |
| "loss": 0.3616, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.29748064279556274, |
| "rewards/margins": 1.576164722442627, |
| "rewards/rejected": -1.2786839008331299, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.4246153846153846, |
| "grad_norm": 24.998257178693006, |
| "learning_rate": 4.81355307410676e-07, |
| "logits/chosen": -1.268651008605957, |
| "logits/rejected": -1.2737505435943604, |
| "logps/chosen": -21.684688568115234, |
| "logps/rejected": -20.43457794189453, |
| "loss": 0.3963, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.701554536819458, |
| "rewards/margins": 1.2370011806488037, |
| "rewards/rejected": -0.5354464650154114, |
| "step": 92 |
| }, |
| { |
| "epoch": 0.4338461538461538, |
| "grad_norm": 30.39233888946852, |
| "learning_rate": 4.79790922448953e-07, |
| "logits/chosen": -1.2319780588150024, |
| "logits/rejected": -1.234665870666504, |
| "logps/chosen": -22.746065139770508, |
| "logps/rejected": -37.10270309448242, |
| "loss": 0.4055, |
| "rewards/accuracies": 0.7916666865348816, |
| "rewards/chosen": 0.33227479457855225, |
| "rewards/margins": 1.4662950038909912, |
| "rewards/rejected": -1.1340200901031494, |
| "step": 94 |
| }, |
| { |
| "epoch": 0.4430769230769231, |
| "grad_norm": 28.351607065877335, |
| "learning_rate": 4.78166278800385e-07, |
| "logits/chosen": -1.2103080749511719, |
| "logits/rejected": -1.2216867208480835, |
| "logps/chosen": -22.36292839050293, |
| "logps/rejected": -36.19468307495117, |
| "loss": 0.3633, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.46569257974624634, |
| "rewards/margins": 1.3663029670715332, |
| "rewards/rejected": -0.9006102681159973, |
| "step": 96 |
| }, |
| { |
| "epoch": 0.4523076923076923, |
| "grad_norm": 27.63597035013981, |
| "learning_rate": 4.7648180249961165e-07, |
| "logits/chosen": -1.2609645128250122, |
| "logits/rejected": -1.2675108909606934, |
| "logps/chosen": -19.6772403717041, |
| "logps/rejected": -22.703941345214844, |
| "loss": 0.3425, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.42960312962532043, |
| "rewards/margins": 1.3149679899215698, |
| "rewards/rejected": -0.8853649497032166, |
| "step": 98 |
| }, |
| { |
| "epoch": 0.46153846153846156, |
| "grad_norm": 27.095171417356656, |
| "learning_rate": 4.747379352713488e-07, |
| "logits/chosen": -1.2016191482543945, |
| "logits/rejected": -1.212724208831787, |
| "logps/chosen": -26.863676071166992, |
| "logps/rejected": -35.31084442138672, |
| "loss": 0.3626, |
| "rewards/accuracies": 0.7638888955116272, |
| "rewards/chosen": 0.43012529611587524, |
| "rewards/margins": 1.344970703125, |
| "rewards/rejected": -0.9148455858230591, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.46153846153846156, |
| "eval_logits/chosen": -1.2631281614303589, |
| "eval_logits/rejected": -1.2726249694824219, |
| "eval_logps/chosen": -22.157392501831055, |
| "eval_logps/rejected": -28.169017791748047, |
| "eval_loss": 0.3646220564842224, |
| "eval_rewards/accuracies": 0.7972350120544434, |
| "eval_rewards/chosen": 0.4446515440940857, |
| "eval_rewards/margins": 1.348763346672058, |
| "eval_rewards/rejected": -0.904111921787262, |
| "eval_runtime": 215.7885, |
| "eval_samples_per_second": 8.036, |
| "eval_steps_per_second": 2.011, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.4707692307692308, |
| "grad_norm": 32.35798457566701, |
| "learning_rate": 4.7293513441455357e-07, |
| "logits/chosen": -1.2197188138961792, |
| "logits/rejected": -1.2320291996002197, |
| "logps/chosen": -19.279041290283203, |
| "logps/rejected": -35.00586700439453, |
| "loss": 0.3714, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.3205001652240753, |
| "rewards/margins": 1.3763878345489502, |
| "rewards/rejected": -1.0558876991271973, |
| "step": 102 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 21.70119714606352, |
| "learning_rate": 4.7107387268250586e-07, |
| "logits/chosen": -1.1967614889144897, |
| "logits/rejected": -1.220970630645752, |
| "logps/chosen": -10.033695220947266, |
| "logps/rejected": -38.51593017578125, |
| "loss": 0.3835, |
| "rewards/accuracies": 0.7777777910232544, |
| "rewards/chosen": 0.4108971059322357, |
| "rewards/margins": 1.6398005485534668, |
| "rewards/rejected": -1.2289036512374878, |
| "step": 104 |
| }, |
| { |
| "epoch": 0.48923076923076925, |
| "grad_norm": 22.839162689384967, |
| "learning_rate": 4.691546381588369e-07, |
| "logits/chosen": -1.2221455574035645, |
| "logits/rejected": -1.2347490787506104, |
| "logps/chosen": -20.123445510864258, |
| "logps/rejected": -34.73093032836914, |
| "loss": 0.3528, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.3299613296985626, |
| "rewards/margins": 1.6646933555603027, |
| "rewards/rejected": -1.3347320556640625, |
| "step": 106 |
| }, |
| { |
| "epoch": 0.49846153846153846, |
| "grad_norm": 30.91989303041632, |
| "learning_rate": 4.6717793412953776e-07, |
| "logits/chosen": -1.2001112699508667, |
| "logits/rejected": -1.2213759422302246, |
| "logps/chosen": -18.639766693115234, |
| "logps/rejected": -38.698211669921875, |
| "loss": 0.3751, |
| "rewards/accuracies": 0.7361111044883728, |
| "rewards/chosen": 0.3170078694820404, |
| "rewards/margins": 1.7733925580978394, |
| "rewards/rejected": -1.456384539604187, |
| "step": 108 |
| }, |
| { |
| "epoch": 0.5076923076923077, |
| "grad_norm": 22.21075058785491, |
| "learning_rate": 4.651442789509813e-07, |
| "logits/chosen": -1.172301173210144, |
| "logits/rejected": -1.1873422861099243, |
| "logps/chosen": -19.037778854370117, |
| "logps/rejected": -35.6918830871582, |
| "loss": 0.3632, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.44801807403564453, |
| "rewards/margins": 1.6537230014801025, |
| "rewards/rejected": -1.2057050466537476, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.5169230769230769, |
| "grad_norm": 22.23191382020911, |
| "learning_rate": 4.630542059139923e-07, |
| "logits/chosen": -1.1621766090393066, |
| "logits/rejected": -1.1781913042068481, |
| "logps/chosen": -26.200401306152344, |
| "logps/rejected": -28.19536590576172, |
| "loss": 0.3117, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.4852801561355591, |
| "rewards/margins": 1.5631003379821777, |
| "rewards/rejected": -1.0778203010559082, |
| "step": 112 |
| }, |
| { |
| "epoch": 0.5261538461538462, |
| "grad_norm": 26.06519967082825, |
| "learning_rate": 4.609082631040011e-07, |
| "logits/chosen": -1.1710741519927979, |
| "logits/rejected": -1.1770610809326172, |
| "logps/chosen": -26.139328002929688, |
| "logps/rejected": -38.44914627075195, |
| "loss": 0.3191, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.42665359377861023, |
| "rewards/margins": 1.9680951833724976, |
| "rewards/rejected": -1.5414414405822754, |
| "step": 114 |
| }, |
| { |
| "epoch": 0.5353846153846153, |
| "grad_norm": 23.76055177774163, |
| "learning_rate": 4.5870701325731773e-07, |
| "logits/chosen": -1.1841078996658325, |
| "logits/rejected": -1.2016386985778809, |
| "logps/chosen": -18.3129940032959, |
| "logps/rejected": -38.7909049987793, |
| "loss": 0.3422, |
| "rewards/accuracies": 0.7638888955116272, |
| "rewards/chosen": 0.31725624203681946, |
| "rewards/margins": 1.8888146877288818, |
| "rewards/rejected": -1.5715583562850952, |
| "step": 116 |
| }, |
| { |
| "epoch": 0.5446153846153846, |
| "grad_norm": 22.451458526325442, |
| "learning_rate": 4.5645103361356407e-07, |
| "logits/chosen": -1.203595519065857, |
| "logits/rejected": -1.1993364095687866, |
| "logps/chosen": -29.456233978271484, |
| "logps/rejected": -24.436891555786133, |
| "loss": 0.3111, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.4006561040878296, |
| "rewards/margins": 1.460686206817627, |
| "rewards/rejected": -1.0600301027297974, |
| "step": 118 |
| }, |
| { |
| "epoch": 0.5538461538461539, |
| "grad_norm": 20.899441336146108, |
| "learning_rate": 4.541409157643027e-07, |
| "logits/chosen": -1.113027811050415, |
| "logits/rejected": -1.1339952945709229, |
| "logps/chosen": -22.780738830566406, |
| "logps/rejected": -37.4469108581543, |
| "loss": 0.263, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.5090766549110413, |
| "rewards/margins": 2.038201332092285, |
| "rewards/rejected": -1.5291246175765991, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.5538461538461539, |
| "eval_logits/chosen": -1.2401551008224487, |
| "eval_logits/rejected": -1.249323844909668, |
| "eval_logps/chosen": -22.120243072509766, |
| "eval_logps/rejected": -28.963603973388672, |
| "eval_loss": 0.32304224371910095, |
| "eval_rewards/accuracies": 0.8122119903564453, |
| "eval_rewards/chosen": 0.46322670578956604, |
| "eval_rewards/margins": 1.764631986618042, |
| "eval_rewards/rejected": -1.3014051914215088, |
| "eval_runtime": 215.8398, |
| "eval_samples_per_second": 8.034, |
| "eval_steps_per_second": 2.011, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.563076923076923, |
| "grad_norm": 25.722122527925197, |
| "learning_rate": 4.517772654979023e-07, |
| "logits/chosen": -1.1628613471984863, |
| "logits/rejected": -1.1666890382766724, |
| "logps/chosen": -28.28006935119629, |
| "logps/rejected": -32.06778335571289, |
| "loss": 0.2967, |
| "rewards/accuracies": 0.8055555820465088, |
| "rewards/chosen": 0.42497023940086365, |
| "rewards/margins": 1.8420732021331787, |
| "rewards/rejected": -1.4171031713485718, |
| "step": 122 |
| }, |
| { |
| "epoch": 0.5723076923076923, |
| "grad_norm": 18.859437245079093, |
| "learning_rate": 4.4936070264068016e-07, |
| "logits/chosen": -1.097366452217102, |
| "logits/rejected": -1.1257672309875488, |
| "logps/chosen": -19.26881217956543, |
| "logps/rejected": -50.698387145996094, |
| "loss": 0.3122, |
| "rewards/accuracies": 0.8055555820465088, |
| "rewards/chosen": 0.3975294530391693, |
| "rewards/margins": 2.376965045928955, |
| "rewards/rejected": -1.9794355630874634, |
| "step": 124 |
| }, |
| { |
| "epoch": 0.5815384615384616, |
| "grad_norm": 24.12611784808478, |
| "learning_rate": 4.468918608943636e-07, |
| "logits/chosen": -1.188425064086914, |
| "logits/rejected": -1.2095468044281006, |
| "logps/chosen": -22.594573974609375, |
| "logps/rejected": -33.808677673339844, |
| "loss": 0.2989, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.4648338854312897, |
| "rewards/margins": 2.128401756286621, |
| "rewards/rejected": -1.6635680198669434, |
| "step": 126 |
| }, |
| { |
| "epoch": 0.5907692307692308, |
| "grad_norm": 21.121113872126465, |
| "learning_rate": 4.443713876699123e-07, |
| "logits/chosen": -1.176856279373169, |
| "logits/rejected": -1.175789713859558, |
| "logps/chosen": -31.682504653930664, |
| "logps/rejected": -26.862850189208984, |
| "loss": 0.2881, |
| "rewards/accuracies": 0.7777777910232544, |
| "rewards/chosen": 0.47753646969795227, |
| "rewards/margins": 1.661524772644043, |
| "rewards/rejected": -1.183988332748413, |
| "step": 128 |
| }, |
| { |
| "epoch": 0.6, |
| "grad_norm": 24.221092280098347, |
| "learning_rate": 4.417999439177465e-07, |
| "logits/chosen": -1.1786390542984009, |
| "logits/rejected": -1.1881896257400513, |
| "logps/chosen": -18.69803237915039, |
| "logps/rejected": -28.687692642211914, |
| "loss": 0.2737, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.5532296895980835, |
| "rewards/margins": 2.0457603931427, |
| "rewards/rejected": -1.4925308227539062, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.6092307692307692, |
| "grad_norm": 19.171893778962126, |
| "learning_rate": 4.391782039544238e-07, |
| "logits/chosen": -1.2097636461257935, |
| "logits/rejected": -1.2146636247634888, |
| "logps/chosen": -19.53115463256836, |
| "logps/rejected": -19.350337982177734, |
| "loss": 0.3284, |
| "rewards/accuracies": 0.7083333134651184, |
| "rewards/chosen": 0.28336918354034424, |
| "rewards/margins": 1.5194146633148193, |
| "rewards/rejected": -1.236045479774475, |
| "step": 132 |
| }, |
| { |
| "epoch": 0.6184615384615385, |
| "grad_norm": 22.368959777821875, |
| "learning_rate": 4.365068552858115e-07, |
| "logits/chosen": -1.2042018175125122, |
| "logits/rejected": -1.2163949012756348, |
| "logps/chosen": -24.11139488220215, |
| "logps/rejected": -33.35640335083008, |
| "loss": 0.3137, |
| "rewards/accuracies": 0.7638888955116272, |
| "rewards/chosen": 0.1719236522912979, |
| "rewards/margins": 1.7209672927856445, |
| "rewards/rejected": -1.5490436553955078, |
| "step": 134 |
| }, |
| { |
| "epoch": 0.6276923076923077, |
| "grad_norm": 17.354174303387865, |
| "learning_rate": 4.337865984268001e-07, |
| "logits/chosen": -1.1561534404754639, |
| "logits/rejected": -1.1622954607009888, |
| "logps/chosen": -15.14254093170166, |
| "logps/rejected": -27.18238067626953, |
| "loss": 0.2954, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.34695935249328613, |
| "rewards/margins": 1.897645115852356, |
| "rewards/rejected": -1.5506855249404907, |
| "step": 136 |
| }, |
| { |
| "epoch": 0.6369230769230769, |
| "grad_norm": 14.475969356318869, |
| "learning_rate": 4.310181467176054e-07, |
| "logits/chosen": -1.1768825054168701, |
| "logits/rejected": -1.1757102012634277, |
| "logps/chosen": -25.93258285522461, |
| "logps/rejected": -32.286590576171875, |
| "loss": 0.2914, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.42600950598716736, |
| "rewards/margins": 2.0175862312316895, |
| "rewards/rejected": -1.5915768146514893, |
| "step": 138 |
| }, |
| { |
| "epoch": 0.6461538461538462, |
| "grad_norm": 18.34569474287581, |
| "learning_rate": 4.282022261367073e-07, |
| "logits/chosen": -1.2166173458099365, |
| "logits/rejected": -1.2223114967346191, |
| "logps/chosen": -20.700721740722656, |
| "logps/rejected": -25.006229400634766, |
| "loss": 0.2717, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.5470355749130249, |
| "rewards/margins": 1.990134358406067, |
| "rewards/rejected": -1.4430986642837524, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.6461538461538462, |
| "eval_logits/chosen": -1.221505880355835, |
| "eval_logits/rejected": -1.2305463552474976, |
| "eval_logps/chosen": -22.114253997802734, |
| "eval_logps/rejected": -29.54737663269043, |
| "eval_loss": 0.29700523614883423, |
| "eval_rewards/accuracies": 0.8179723620414734, |
| "eval_rewards/chosen": 0.46622127294540405, |
| "eval_rewards/margins": 2.0595133304595947, |
| "eval_rewards/rejected": -1.5932921171188354, |
| "eval_runtime": 215.9245, |
| "eval_samples_per_second": 8.031, |
| "eval_steps_per_second": 2.01, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.6553846153846153, |
| "grad_norm": 24.003361700026115, |
| "learning_rate": 4.253395751104748e-07, |
| "logits/chosen": -1.2128342390060425, |
| "logits/rejected": -1.2202144861221313, |
| "logps/chosen": -20.926525115966797, |
| "logps/rejected": -33.759159088134766, |
| "loss": 0.2796, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.4563888907432556, |
| "rewards/margins": 2.332362413406372, |
| "rewards/rejected": -1.8759733438491821, |
| "step": 142 |
| }, |
| { |
| "epoch": 0.6646153846153846, |
| "grad_norm": 22.96956018291041, |
| "learning_rate": 4.2243094431952607e-07, |
| "logits/chosen": -1.1733120679855347, |
| "logits/rejected": -1.1876205205917358, |
| "logps/chosen": -20.787324905395508, |
| "logps/rejected": -44.41487503051758, |
| "loss": 0.2904, |
| "rewards/accuracies": 0.7638888955116272, |
| "rewards/chosen": 0.4227790832519531, |
| "rewards/margins": 2.513406753540039, |
| "rewards/rejected": -2.090627431869507, |
| "step": 144 |
| }, |
| { |
| "epoch": 0.6738461538461539, |
| "grad_norm": 20.337910027315395, |
| "learning_rate": 4.194770965018758e-07, |
| "logits/chosen": -1.1829084157943726, |
| "logits/rejected": -1.1901525259017944, |
| "logps/chosen": -22.88217544555664, |
| "logps/rejected": -40.51693344116211, |
| "loss": 0.2982, |
| "rewards/accuracies": 0.7777777910232544, |
| "rewards/chosen": 0.32644984126091003, |
| "rewards/margins": 2.2273294925689697, |
| "rewards/rejected": -1.9008797407150269, |
| "step": 146 |
| }, |
| { |
| "epoch": 0.683076923076923, |
| "grad_norm": 16.955507402789948, |
| "learning_rate": 4.1647880625292027e-07, |
| "logits/chosen": -1.1585676670074463, |
| "logits/rejected": -1.1673483848571777, |
| "logps/chosen": -17.565954208374023, |
| "logps/rejected": -30.01752471923828, |
| "loss": 0.2381, |
| "rewards/accuracies": 0.9027777910232544, |
| "rewards/chosen": 0.6770419478416443, |
| "rewards/margins": 2.5649421215057373, |
| "rewards/rejected": -1.8879002332687378, |
| "step": 148 |
| }, |
| { |
| "epoch": 0.6923076923076923, |
| "grad_norm": 16.268353553690783, |
| "learning_rate": 4.1343685982231315e-07, |
| "logits/chosen": -1.2300368547439575, |
| "logits/rejected": -1.2412070035934448, |
| "logps/chosen": -19.158246994018555, |
| "logps/rejected": -30.00787353515625, |
| "loss": 0.2576, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.28651073575019836, |
| "rewards/margins": 2.1342878341674805, |
| "rewards/rejected": -1.8477774858474731, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.7015384615384616, |
| "grad_norm": 22.707867679754226, |
| "learning_rate": 4.1035205490778496e-07, |
| "logits/chosen": -1.1675605773925781, |
| "logits/rejected": -1.1745511293411255, |
| "logps/chosen": -24.983802795410156, |
| "logps/rejected": -32.00082015991211, |
| "loss": 0.3007, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.4517359137535095, |
| "rewards/margins": 2.2256662845611572, |
| "rewards/rejected": -1.7739304304122925, |
| "step": 152 |
| }, |
| { |
| "epoch": 0.7107692307692308, |
| "grad_norm": 17.503865371681442, |
| "learning_rate": 4.072252004459611e-07, |
| "logits/chosen": -1.1371846199035645, |
| "logits/rejected": -1.1358321905136108, |
| "logps/chosen": -26.079011917114258, |
| "logps/rejected": -27.951416015625, |
| "loss": 0.2471, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.44966569542884827, |
| "rewards/margins": 2.104396104812622, |
| "rewards/rejected": -1.6547303199768066, |
| "step": 154 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 15.32657259953523, |
| "learning_rate": 4.040571164002318e-07, |
| "logits/chosen": -1.189456820487976, |
| "logits/rejected": -1.1948577165603638, |
| "logps/chosen": -20.083751678466797, |
| "logps/rejected": -30.10634994506836, |
| "loss": 0.2351, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.42890670895576477, |
| "rewards/margins": 2.341860771179199, |
| "rewards/rejected": -1.9129540920257568, |
| "step": 156 |
| }, |
| { |
| "epoch": 0.7292307692307692, |
| "grad_norm": 17.946669808646828, |
| "learning_rate": 4.0084863354573116e-07, |
| "logits/chosen": -1.1215004920959473, |
| "logits/rejected": -1.1300181150436401, |
| "logps/chosen": -23.436655044555664, |
| "logps/rejected": -34.97710418701172, |
| "loss": 0.2706, |
| "rewards/accuracies": 0.9444444179534912, |
| "rewards/chosen": 0.22723568975925446, |
| "rewards/margins": 2.1446826457977295, |
| "rewards/rejected": -1.9174467325210571, |
| "step": 158 |
| }, |
| { |
| "epoch": 0.7384615384615385, |
| "grad_norm": 16.72039592892195, |
| "learning_rate": 3.9760059325148063e-07, |
| "logits/chosen": -1.2237818241119385, |
| "logits/rejected": -1.2211045026779175, |
| "logps/chosen": -24.31806755065918, |
| "logps/rejected": -25.250701904296875, |
| "loss": 0.2351, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.4868224859237671, |
| "rewards/margins": 2.124577522277832, |
| "rewards/rejected": -1.637755274772644, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.7384615384615385, |
| "eval_logits/chosen": -1.2072025537490845, |
| "eval_logits/rejected": -1.216115951538086, |
| "eval_logps/chosen": -22.174776077270508, |
| "eval_logps/rejected": -30.134973526000977, |
| "eval_loss": 0.27949145436286926, |
| "eval_rewards/accuracies": 0.8248847723007202, |
| "eval_rewards/chosen": 0.4359608590602875, |
| "eval_rewards/margins": 2.3230507373809814, |
| "eval_rewards/rejected": -1.8870899677276611, |
| "eval_runtime": 216.1181, |
| "eval_samples_per_second": 8.023, |
| "eval_steps_per_second": 2.008, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.7476923076923077, |
| "grad_norm": 16.877732796497064, |
| "learning_rate": 3.9431384725975485e-07, |
| "logits/chosen": -1.1728930473327637, |
| "logits/rejected": -1.1828408241271973, |
| "logps/chosen": -20.051979064941406, |
| "logps/rejected": -30.078739166259766, |
| "loss": 0.2806, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.4627165198326111, |
| "rewards/margins": 2.1041107177734375, |
| "rewards/rejected": -1.641394019126892, |
| "step": 162 |
| }, |
| { |
| "epoch": 0.7569230769230769, |
| "grad_norm": 17.236677422360824, |
| "learning_rate": 3.909892574627266e-07, |
| "logits/chosen": -1.1840589046478271, |
| "logits/rejected": -1.205323338508606, |
| "logps/chosen": -20.25952911376953, |
| "logps/rejected": -43.16006851196289, |
| "loss": 0.267, |
| "rewards/accuracies": 0.8055555820465088, |
| "rewards/chosen": 0.34341666102409363, |
| "rewards/margins": 2.8926875591278076, |
| "rewards/rejected": -2.5492708683013916, |
| "step": 164 |
| }, |
| { |
| "epoch": 0.7661538461538462, |
| "grad_norm": 15.084626056041332, |
| "learning_rate": 3.876276956764509e-07, |
| "logits/chosen": -1.172157883644104, |
| "logits/rejected": -1.1869869232177734, |
| "logps/chosen": -20.39401626586914, |
| "logps/rejected": -35.54499816894531, |
| "loss": 0.2191, |
| "rewards/accuracies": 0.9166666865348816, |
| "rewards/chosen": 0.533491313457489, |
| "rewards/margins": 3.2933194637298584, |
| "rewards/rejected": -2.7598280906677246, |
| "step": 166 |
| }, |
| { |
| "epoch": 0.7753846153846153, |
| "grad_norm": 16.522846792297653, |
| "learning_rate": 3.8423004341224595e-07, |
| "logits/chosen": -1.1675995588302612, |
| "logits/rejected": -1.1726378202438354, |
| "logps/chosen": -22.266756057739258, |
| "logps/rejected": -27.90992546081543, |
| "loss": 0.2137, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.3478531837463379, |
| "rewards/margins": 2.3764336109161377, |
| "rewards/rejected": -2.028580665588379, |
| "step": 168 |
| }, |
| { |
| "epoch": 0.7846153846153846, |
| "grad_norm": 18.709310219062342, |
| "learning_rate": 3.807971916455325e-07, |
| "logits/chosen": -1.1257578134536743, |
| "logits/rejected": -1.1353437900543213, |
| "logps/chosen": -25.48769187927246, |
| "logps/rejected": -37.34423065185547, |
| "loss": 0.2439, |
| "rewards/accuracies": 0.7777777910232544, |
| "rewards/chosen": 0.30796098709106445, |
| "rewards/margins": 2.5804708003997803, |
| "rewards/rejected": -2.2725095748901367, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.7938461538461539, |
| "grad_norm": 18.811516964897933, |
| "learning_rate": 3.773300405821908e-07, |
| "logits/chosen": -1.2032923698425293, |
| "logits/rejected": -1.1944453716278076, |
| "logps/chosen": -22.42747688293457, |
| "logps/rejected": -24.809179306030273, |
| "loss": 0.2706, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.4598681628704071, |
| "rewards/margins": 2.331010103225708, |
| "rewards/rejected": -1.871142029762268, |
| "step": 172 |
| }, |
| { |
| "epoch": 0.803076923076923, |
| "grad_norm": 27.213611533570646, |
| "learning_rate": 3.738294994224969e-07, |
| "logits/chosen": -1.1406216621398926, |
| "logits/rejected": -1.1456246376037598, |
| "logps/chosen": -22.41916847229004, |
| "logps/rejected": -25.79179573059082, |
| "loss": 0.2525, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.5410938858985901, |
| "rewards/margins": 2.5380003452301025, |
| "rewards/rejected": -1.9969062805175781, |
| "step": 174 |
| }, |
| { |
| "epoch": 0.8123076923076923, |
| "grad_norm": 22.120419375719585, |
| "learning_rate": 3.7029648612270123e-07, |
| "logits/chosen": -1.1604636907577515, |
| "logits/rejected": -1.166500210762024, |
| "logps/chosen": -23.140409469604492, |
| "logps/rejected": -32.539859771728516, |
| "loss": 0.2445, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.5552553534507751, |
| "rewards/margins": 2.451958656311035, |
| "rewards/rejected": -1.8967031240463257, |
| "step": 176 |
| }, |
| { |
| "epoch": 0.8215384615384616, |
| "grad_norm": 23.529456123726142, |
| "learning_rate": 3.6673192715431014e-07, |
| "logits/chosen": -1.172749638557434, |
| "logits/rejected": -1.1873490810394287, |
| "logps/chosen": -19.344928741455078, |
| "logps/rejected": -46.30924987792969, |
| "loss": 0.2576, |
| "rewards/accuracies": 0.8055555820465088, |
| "rewards/chosen": 0.3556906580924988, |
| "rewards/margins": 3.198575973510742, |
| "rewards/rejected": -2.8428850173950195, |
| "step": 178 |
| }, |
| { |
| "epoch": 0.8307692307692308, |
| "grad_norm": 16.07954647927614, |
| "learning_rate": 3.6313675726113475e-07, |
| "logits/chosen": -1.1696263551712036, |
| "logits/rejected": -1.1719523668289185, |
| "logps/chosen": -24.40313148498535, |
| "logps/rejected": -30.179893493652344, |
| "loss": 0.2373, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.5325056314468384, |
| "rewards/margins": 2.6024298667907715, |
| "rewards/rejected": -2.0699243545532227, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.8307692307692308, |
| "eval_logits/chosen": -1.1957546472549438, |
| "eval_logits/rejected": -1.2044621706008911, |
| "eval_logps/chosen": -22.226091384887695, |
| "eval_logps/rejected": -30.679323196411133, |
| "eval_loss": 0.2662460505962372, |
| "eval_rewards/accuracies": 0.8271889686584473, |
| "eval_rewards/chosen": 0.4103015661239624, |
| "eval_rewards/margins": 2.569566011428833, |
| "eval_rewards/rejected": -2.15926456451416, |
| "eval_runtime": 216.1605, |
| "eval_samples_per_second": 8.022, |
| "eval_steps_per_second": 2.008, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.84, |
| "grad_norm": 12.027824441881227, |
| "learning_rate": 3.595119192141706e-07, |
| "logits/chosen": -1.1798688173294067, |
| "logits/rejected": -1.190478801727295, |
| "logps/chosen": -23.84467315673828, |
| "logps/rejected": -27.77214241027832, |
| "loss": 0.1945, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.5185620784759521, |
| "rewards/margins": 2.7370386123657227, |
| "rewards/rejected": -2.2184765338897705, |
| "step": 182 |
| }, |
| { |
| "epoch": 0.8492307692307692, |
| "grad_norm": 21.657852790803656, |
| "learning_rate": 3.558583635643726e-07, |
| "logits/chosen": -1.1619257926940918, |
| "logits/rejected": -1.1783702373504639, |
| "logps/chosen": -20.357545852661133, |
| "logps/rejected": -36.6799430847168, |
| "loss": 0.2859, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.39101898670196533, |
| "rewards/margins": 2.5226354598999023, |
| "rewards/rejected": -2.1316165924072266, |
| "step": 184 |
| }, |
| { |
| "epoch": 0.8584615384615385, |
| "grad_norm": 15.850729398525738, |
| "learning_rate": 3.5217704839338905e-07, |
| "logits/chosen": -1.2039780616760254, |
| "logits/rejected": -1.2015321254730225, |
| "logps/chosen": -25.71788787841797, |
| "logps/rejected": -29.20301628112793, |
| "loss": 0.2245, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.43592390418052673, |
| "rewards/margins": 2.691300392150879, |
| "rewards/rejected": -2.2553763389587402, |
| "step": 186 |
| }, |
| { |
| "epoch": 0.8676923076923077, |
| "grad_norm": 20.33987602806827, |
| "learning_rate": 3.484689390623218e-07, |
| "logits/chosen": -1.173121452331543, |
| "logits/rejected": -1.1853346824645996, |
| "logps/chosen": -21.594472885131836, |
| "logps/rejected": -36.92512130737305, |
| "loss": 0.2243, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.22467082738876343, |
| "rewards/margins": 2.8943564891815186, |
| "rewards/rejected": -2.6696856021881104, |
| "step": 188 |
| }, |
| { |
| "epoch": 0.8769230769230769, |
| "grad_norm": 15.456781978721555, |
| "learning_rate": 3.447350079585767e-07, |
| "logits/chosen": -1.20560884475708, |
| "logits/rejected": -1.2095773220062256, |
| "logps/chosen": -18.067840576171875, |
| "logps/rejected": -24.3345890045166, |
| "loss": 0.2124, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.2674013674259186, |
| "rewards/margins": 2.3308472633361816, |
| "rewards/rejected": -2.063445568084717, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.8861538461538462, |
| "grad_norm": 24.575966523755373, |
| "learning_rate": 3.409762342408719e-07, |
| "logits/chosen": -1.1767027378082275, |
| "logits/rejected": -1.1829452514648438, |
| "logps/chosen": -23.147159576416016, |
| "logps/rejected": -38.63761901855469, |
| "loss": 0.3063, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.2949807345867157, |
| "rewards/margins": 2.8994204998016357, |
| "rewards/rejected": -2.6044397354125977, |
| "step": 192 |
| }, |
| { |
| "epoch": 0.8953846153846153, |
| "grad_norm": 13.903082439233941, |
| "learning_rate": 3.3719360358247053e-07, |
| "logits/chosen": -1.1678471565246582, |
| "logits/rejected": -1.1855759620666504, |
| "logps/chosen": -19.064098358154297, |
| "logps/rejected": -36.09113693237305, |
| "loss": 0.288, |
| "rewards/accuracies": 0.7916666865348816, |
| "rewards/chosen": 0.4278064966201782, |
| "rewards/margins": 2.7983500957489014, |
| "rewards/rejected": -2.3705434799194336, |
| "step": 194 |
| }, |
| { |
| "epoch": 0.9046153846153846, |
| "grad_norm": 16.97717210575951, |
| "learning_rate": 3.3338810791270517e-07, |
| "logits/chosen": -1.1488627195358276, |
| "logits/rejected": -1.161072015762329, |
| "logps/chosen": -16.16121482849121, |
| "logps/rejected": -35.24711608886719, |
| "loss": 0.2587, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.2668210566043854, |
| "rewards/margins": 2.758829116821289, |
| "rewards/rejected": -2.4920082092285156, |
| "step": 196 |
| }, |
| { |
| "epoch": 0.9138461538461539, |
| "grad_norm": 21.684346277519417, |
| "learning_rate": 3.29560745156861e-07, |
| "logits/chosen": -1.1681840419769287, |
| "logits/rejected": -1.1707243919372559, |
| "logps/chosen": -27.238510131835938, |
| "logps/rejected": -29.843427658081055, |
| "loss": 0.2945, |
| "rewards/accuracies": 0.7916666865348816, |
| "rewards/chosen": 0.487039715051651, |
| "rewards/margins": 2.7937545776367188, |
| "rewards/rejected": -2.3067147731781006, |
| "step": 198 |
| }, |
| { |
| "epoch": 0.9230769230769231, |
| "grad_norm": 15.010044100424757, |
| "learning_rate": 3.2571251897448763e-07, |
| "logits/chosen": -1.1483420133590698, |
| "logits/rejected": -1.172219157218933, |
| "logps/chosen": -20.701204299926758, |
| "logps/rejected": -47.092777252197266, |
| "loss": 0.2393, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.5242102742195129, |
| "rewards/margins": 3.446150302886963, |
| "rewards/rejected": -2.9219398498535156, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.9230769230769231, |
| "eval_logits/chosen": -1.188868761062622, |
| "eval_logits/rejected": -1.1974678039550781, |
| "eval_logps/chosen": -22.205198287963867, |
| "eval_logps/rejected": -30.90268325805664, |
| "eval_loss": 0.25766730308532715, |
| "eval_rewards/accuracies": 0.8306451439857483, |
| "eval_rewards/chosen": 0.42075031995773315, |
| "eval_rewards/margins": 2.6916959285736084, |
| "eval_rewards/rejected": -2.2709455490112305, |
| "eval_runtime": 216.204, |
| "eval_samples_per_second": 8.02, |
| "eval_steps_per_second": 2.007, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.9323076923076923, |
| "grad_norm": 24.918463307740545, |
| "learning_rate": 3.218444384962071e-07, |
| "logits/chosen": -1.1572585105895996, |
| "logits/rejected": -1.1649041175842285, |
| "logps/chosen": -20.337928771972656, |
| "logps/rejected": -25.251022338867188, |
| "loss": 0.2872, |
| "rewards/accuracies": 0.7777777910232544, |
| "rewards/chosen": 0.051526255905628204, |
| "rewards/margins": 2.169602155685425, |
| "rewards/rejected": -2.1180758476257324, |
| "step": 202 |
| }, |
| { |
| "epoch": 0.9415384615384615, |
| "grad_norm": 17.132653548760572, |
| "learning_rate": 3.179575180590857e-07, |
| "logits/chosen": -1.1708558797836304, |
| "logits/rejected": -1.1774191856384277, |
| "logps/chosen": -16.72760772705078, |
| "logps/rejected": -29.532522201538086, |
| "loss": 0.2703, |
| "rewards/accuracies": 0.8055555820465088, |
| "rewards/chosen": 0.3555985391139984, |
| "rewards/margins": 2.5367255210876465, |
| "rewards/rejected": -2.1811270713806152, |
| "step": 204 |
| }, |
| { |
| "epoch": 0.9507692307692308, |
| "grad_norm": 18.808695685272248, |
| "learning_rate": 3.1405277694064305e-07, |
| "logits/chosen": -1.13996422290802, |
| "logits/rejected": -1.1603398323059082, |
| "logps/chosen": -20.1070613861084, |
| "logps/rejected": -43.8044319152832, |
| "loss": 0.2133, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.3765062689781189, |
| "rewards/margins": 3.3217618465423584, |
| "rewards/rejected": -2.9452552795410156, |
| "step": 206 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 29.593271367025817, |
| "learning_rate": 3.101312390915634e-07, |
| "logits/chosen": -1.1117515563964844, |
| "logits/rejected": -1.1254826784133911, |
| "logps/chosen": -18.95772933959961, |
| "logps/rejected": -38.70570373535156, |
| "loss": 0.2626, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.19062408804893494, |
| "rewards/margins": 2.819202423095703, |
| "rewards/rejected": -2.6285784244537354, |
| "step": 208 |
| }, |
| { |
| "epoch": 0.9692307692307692, |
| "grad_norm": 19.2158248846026, |
| "learning_rate": 3.0619393286718237e-07, |
| "logits/chosen": -1.1758193969726562, |
| "logits/rejected": -1.18528413772583, |
| "logps/chosen": -25.30388069152832, |
| "logps/rejected": -24.64061737060547, |
| "loss": 0.2715, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.22175876796245575, |
| "rewards/margins": 2.10679292678833, |
| "rewards/rejected": -1.8850340843200684, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.9784615384615385, |
| "grad_norm": 23.720067200725047, |
| "learning_rate": 3.022418907578188e-07, |
| "logits/chosen": -1.1191242933273315, |
| "logits/rejected": -1.1329889297485352, |
| "logps/chosen": -25.677099227905273, |
| "logps/rejected": -39.06088638305664, |
| "loss": 0.2898, |
| "rewards/accuracies": 0.7916666865348816, |
| "rewards/chosen": 0.20263215899467468, |
| "rewards/margins": 3.0255513191223145, |
| "rewards/rejected": -2.8229193687438965, |
| "step": 212 |
| }, |
| { |
| "epoch": 0.9876923076923076, |
| "grad_norm": 15.354779350521344, |
| "learning_rate": 2.98276149118022e-07, |
| "logits/chosen": -1.1088786125183105, |
| "logits/rejected": -1.1292033195495605, |
| "logps/chosen": -24.54433250427246, |
| "logps/rejected": -38.054649353027344, |
| "loss": 0.2164, |
| "rewards/accuracies": 0.9027777910232544, |
| "rewards/chosen": 0.5917240381240845, |
| "rewards/margins": 3.370425224304199, |
| "rewards/rejected": -2.7787015438079834, |
| "step": 214 |
| }, |
| { |
| "epoch": 0.9969230769230769, |
| "grad_norm": 15.922459499539187, |
| "learning_rate": 2.942977478948057e-07, |
| "logits/chosen": -1.134361743927002, |
| "logits/rejected": -1.1381641626358032, |
| "logps/chosen": -29.736419677734375, |
| "logps/rejected": -34.28538513183594, |
| "loss": 0.209, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.47491705417633057, |
| "rewards/margins": 3.0054473876953125, |
| "rewards/rejected": -2.5305304527282715, |
| "step": 216 |
| }, |
| { |
| "epoch": 1.0061538461538462, |
| "grad_norm": 14.602088714669993, |
| "learning_rate": 2.903077303549399e-07, |
| "logits/chosen": -1.1926045417785645, |
| "logits/rejected": -1.2005811929702759, |
| "logps/chosen": -21.338937759399414, |
| "logps/rejected": -31.98470115661621, |
| "loss": 0.2114, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.49925586581230164, |
| "rewards/margins": 3.034120559692383, |
| "rewards/rejected": -2.534864664077759, |
| "step": 218 |
| }, |
| { |
| "epoch": 1.0153846153846153, |
| "grad_norm": 12.776565445469831, |
| "learning_rate": 2.863071428113726e-07, |
| "logits/chosen": -1.180498719215393, |
| "logits/rejected": -1.1876842975616455, |
| "logps/chosen": -21.977970123291016, |
| "logps/rejected": -26.06908416748047, |
| "loss": 0.2223, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.23817205429077148, |
| "rewards/margins": 2.4826109409332275, |
| "rewards/rejected": -2.244438409805298, |
| "step": 220 |
| }, |
| { |
| "epoch": 1.0153846153846153, |
| "eval_logits/chosen": -1.1809991598129272, |
| "eval_logits/rejected": -1.189637303352356, |
| "eval_logps/chosen": -22.231857299804688, |
| "eval_logps/rejected": -31.20700454711914, |
| "eval_loss": 0.25129908323287964, |
| "eval_rewards/accuracies": 0.8329492807388306, |
| "eval_rewards/chosen": 0.4074196219444275, |
| "eval_rewards/margins": 2.8305253982543945, |
| "eval_rewards/rejected": -2.4231057167053223, |
| "eval_runtime": 216.0555, |
| "eval_samples_per_second": 8.026, |
| "eval_steps_per_second": 2.009, |
| "step": 220 |
| }, |
| { |
| "epoch": 1.0246153846153847, |
| "grad_norm": 14.54877776678067, |
| "learning_rate": 2.822970343488516e-07, |
| "logits/chosen": -1.1495935916900635, |
| "logits/rejected": -1.1574082374572754, |
| "logps/chosen": -25.172189712524414, |
| "logps/rejected": -33.7739372253418, |
| "loss": 0.224, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.4095478355884552, |
| "rewards/margins": 2.9969334602355957, |
| "rewards/rejected": -2.587385416030884, |
| "step": 222 |
| }, |
| { |
| "epoch": 1.0338461538461539, |
| "grad_norm": 12.987637533805088, |
| "learning_rate": 2.782784565488211e-07, |
| "logits/chosen": -1.09419846534729, |
| "logits/rejected": -1.1150177717208862, |
| "logps/chosen": -21.80037498474121, |
| "logps/rejected": -47.742916107177734, |
| "loss": 0.2056, |
| "rewards/accuracies": 0.9305555820465088, |
| "rewards/chosen": 0.49535179138183594, |
| "rewards/margins": 4.081587314605713, |
| "rewards/rejected": -3.5862362384796143, |
| "step": 224 |
| }, |
| { |
| "epoch": 1.043076923076923, |
| "grad_norm": 12.537917774467841, |
| "learning_rate": 2.7425246321366205e-07, |
| "logits/chosen": -1.1532597541809082, |
| "logits/rejected": -1.1558729410171509, |
| "logps/chosen": -23.903770446777344, |
| "logps/rejected": -22.89252471923828, |
| "loss": 0.2188, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.5978649258613586, |
| "rewards/margins": 2.4770026206970215, |
| "rewards/rejected": -1.8791378736495972, |
| "step": 226 |
| }, |
| { |
| "epoch": 1.0523076923076924, |
| "grad_norm": 11.390266637295149, |
| "learning_rate": 2.7022011009035107e-07, |
| "logits/chosen": -1.1780048608779907, |
| "logits/rejected": -1.1780657768249512, |
| "logps/chosen": -20.99365997314453, |
| "logps/rejected": -35.256507873535156, |
| "loss": 0.1785, |
| "rewards/accuracies": 0.9166666865348816, |
| "rewards/chosen": 0.25620290637016296, |
| "rewards/margins": 3.1927871704101562, |
| "rewards/rejected": -2.936584234237671, |
| "step": 228 |
| }, |
| { |
| "epoch": 1.0615384615384615, |
| "grad_norm": 13.274197122497501, |
| "learning_rate": 2.661824545936089e-07, |
| "logits/chosen": -1.1301528215408325, |
| "logits/rejected": -1.141854166984558, |
| "logps/chosen": -22.90785789489746, |
| "logps/rejected": -39.776309967041016, |
| "loss": 0.1848, |
| "rewards/accuracies": 0.9305555820465088, |
| "rewards/chosen": 0.23726129531860352, |
| "rewards/margins": 3.6220147609710693, |
| "rewards/rejected": -3.3847532272338867, |
| "step": 230 |
| }, |
| { |
| "epoch": 1.0707692307692307, |
| "grad_norm": 11.899842789993972, |
| "learning_rate": 2.621405555286121e-07, |
| "logits/chosen": -1.1494054794311523, |
| "logits/rejected": -1.158327579498291, |
| "logps/chosen": -27.49151611328125, |
| "logps/rejected": -33.164703369140625, |
| "loss": 0.1801, |
| "rewards/accuracies": 0.9166666865348816, |
| "rewards/chosen": 0.473955363035202, |
| "rewards/margins": 3.1881282329559326, |
| "rewards/rejected": -2.7141730785369873, |
| "step": 232 |
| }, |
| { |
| "epoch": 1.08, |
| "grad_norm": 12.024964222481547, |
| "learning_rate": 2.58095472813339e-07, |
| "logits/chosen": -1.1302716732025146, |
| "logits/rejected": -1.1499823331832886, |
| "logps/chosen": -25.619178771972656, |
| "logps/rejected": -35.781768798828125, |
| "loss": 0.1808, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.593082070350647, |
| "rewards/margins": 3.549994468688965, |
| "rewards/rejected": -2.9569127559661865, |
| "step": 234 |
| }, |
| { |
| "epoch": 1.0892307692307692, |
| "grad_norm": 16.982420323384893, |
| "learning_rate": 2.540482672006254e-07, |
| "logits/chosen": -1.1983014345169067, |
| "logits/rejected": -1.2088627815246582, |
| "logps/chosen": -20.2447566986084, |
| "logps/rejected": -33.8237419128418, |
| "loss": 0.2502, |
| "rewards/accuracies": 0.8055555820465088, |
| "rewards/chosen": 0.2588607966899872, |
| "rewards/margins": 2.6979219913482666, |
| "rewards/rejected": -2.439061164855957, |
| "step": 236 |
| }, |
| { |
| "epoch": 1.0984615384615384, |
| "grad_norm": 14.78335151339772, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -1.1217488050460815, |
| "logits/rejected": -1.126597285270691, |
| "logps/chosen": -24.313417434692383, |
| "logps/rejected": -32.5634880065918, |
| "loss": 0.1857, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.4129423499107361, |
| "rewards/margins": 3.0672991275787354, |
| "rewards/rejected": -2.6543567180633545, |
| "step": 238 |
| }, |
| { |
| "epoch": 1.1076923076923078, |
| "grad_norm": 9.560418611995035, |
| "learning_rate": 2.459517327993746e-07, |
| "logits/chosen": -1.1439785957336426, |
| "logits/rejected": -1.1501950025558472, |
| "logps/chosen": -21.520601272583008, |
| "logps/rejected": -36.128475189208984, |
| "loss": 0.1631, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.16983138024806976, |
| "rewards/margins": 3.335303544998169, |
| "rewards/rejected": -3.1654722690582275, |
| "step": 240 |
| }, |
| { |
| "epoch": 1.1076923076923078, |
| "eval_logits/chosen": -1.1771941184997559, |
| "eval_logits/rejected": -1.1856648921966553, |
| "eval_logps/chosen": -22.31366539001465, |
| "eval_logps/rejected": -31.599573135375977, |
| "eval_loss": 0.24783480167388916, |
| "eval_rewards/accuracies": 0.8317972421646118, |
| "eval_rewards/chosen": 0.3665139377117157, |
| "eval_rewards/margins": 2.98590350151062, |
| "eval_rewards/rejected": -2.619389295578003, |
| "eval_runtime": 216.1562, |
| "eval_samples_per_second": 8.022, |
| "eval_steps_per_second": 2.008, |
| "step": 240 |
| }, |
| { |
| "epoch": 1.116923076923077, |
| "grad_norm": 13.013402968505392, |
| "learning_rate": 2.4190452718666105e-07, |
| "logits/chosen": -1.0899126529693604, |
| "logits/rejected": -1.1027652025222778, |
| "logps/chosen": -15.734682083129883, |
| "logps/rejected": -27.53190803527832, |
| "loss": 0.2287, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.5433827638626099, |
| "rewards/margins": 3.0215795040130615, |
| "rewards/rejected": -2.478196859359741, |
| "step": 242 |
| }, |
| { |
| "epoch": 1.126153846153846, |
| "grad_norm": 12.301318346382136, |
| "learning_rate": 2.37859444471388e-07, |
| "logits/chosen": -1.1361184120178223, |
| "logits/rejected": -1.151028037071228, |
| "logps/chosen": -24.852954864501953, |
| "logps/rejected": -40.693912506103516, |
| "loss": 0.1914, |
| "rewards/accuracies": 0.9166666865348816, |
| "rewards/chosen": 0.489397794008255, |
| "rewards/margins": 3.448162794113159, |
| "rewards/rejected": -2.9587647914886475, |
| "step": 244 |
| }, |
| { |
| "epoch": 1.1353846153846154, |
| "grad_norm": 13.708460236846275, |
| "learning_rate": 2.3381754540639106e-07, |
| "logits/chosen": -1.1237130165100098, |
| "logits/rejected": -1.1399991512298584, |
| "logps/chosen": -21.652952194213867, |
| "logps/rejected": -30.665048599243164, |
| "loss": 0.2272, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.5611749291419983, |
| "rewards/margins": 3.155482292175293, |
| "rewards/rejected": -2.5943074226379395, |
| "step": 246 |
| }, |
| { |
| "epoch": 1.1446153846153846, |
| "grad_norm": 11.563478452101487, |
| "learning_rate": 2.2977988990964896e-07, |
| "logits/chosen": -1.0979208946228027, |
| "logits/rejected": -1.111803650856018, |
| "logps/chosen": -21.861614227294922, |
| "logps/rejected": -38.676361083984375, |
| "loss": 0.2243, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.13799840211868286, |
| "rewards/margins": 3.1060800552368164, |
| "rewards/rejected": -2.968081474304199, |
| "step": 248 |
| }, |
| { |
| "epoch": 1.1538461538461537, |
| "grad_norm": 12.63303273344697, |
| "learning_rate": 2.2574753678633798e-07, |
| "logits/chosen": -1.2150633335113525, |
| "logits/rejected": -1.2195019721984863, |
| "logps/chosen": -19.639219284057617, |
| "logps/rejected": -22.85377311706543, |
| "loss": 0.2111, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.478664755821228, |
| "rewards/margins": 2.8225910663604736, |
| "rewards/rejected": -2.343926429748535, |
| "step": 250 |
| }, |
| { |
| "epoch": 1.1630769230769231, |
| "grad_norm": 15.55104305702512, |
| "learning_rate": 2.2172154345117894e-07, |
| "logits/chosen": -1.1489689350128174, |
| "logits/rejected": -1.1607710123062134, |
| "logps/chosen": -22.335952758789062, |
| "logps/rejected": -43.476783752441406, |
| "loss": 0.1866, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.6636537909507751, |
| "rewards/margins": 4.2835187911987305, |
| "rewards/rejected": -3.6198649406433105, |
| "step": 252 |
| }, |
| { |
| "epoch": 1.1723076923076923, |
| "grad_norm": 19.58611284576425, |
| "learning_rate": 2.1770296565114846e-07, |
| "logits/chosen": -1.174638271331787, |
| "logits/rejected": -1.1910815238952637, |
| "logps/chosen": -19.441059112548828, |
| "logps/rejected": -23.29158592224121, |
| "loss": 0.2382, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.15968316793441772, |
| "rewards/margins": 2.543644428253174, |
| "rewards/rejected": -2.3839612007141113, |
| "step": 254 |
| }, |
| { |
| "epoch": 1.1815384615384614, |
| "grad_norm": 14.83480005382789, |
| "learning_rate": 2.1369285718862748e-07, |
| "logits/chosen": -1.0653572082519531, |
| "logits/rejected": -1.0726639032363892, |
| "logps/chosen": -24.378429412841797, |
| "logps/rejected": -48.50611877441406, |
| "loss": 0.1932, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.2468690127134323, |
| "rewards/margins": 4.218470096588135, |
| "rewards/rejected": -3.9716007709503174, |
| "step": 256 |
| }, |
| { |
| "epoch": 1.1907692307692308, |
| "grad_norm": 14.627626741140055, |
| "learning_rate": 2.0969226964506005e-07, |
| "logits/chosen": -1.1564842462539673, |
| "logits/rejected": -1.1586439609527588, |
| "logps/chosen": -25.08201789855957, |
| "logps/rejected": -26.51468849182129, |
| "loss": 0.2157, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.42589980363845825, |
| "rewards/margins": 3.205916166305542, |
| "rewards/rejected": -2.7800166606903076, |
| "step": 258 |
| }, |
| { |
| "epoch": 1.2, |
| "grad_norm": 13.600232617567109, |
| "learning_rate": 2.0570225210519433e-07, |
| "logits/chosen": -1.1147321462631226, |
| "logits/rejected": -1.1307651996612549, |
| "logps/chosen": -22.724639892578125, |
| "logps/rejected": -38.13914489746094, |
| "loss": 0.1956, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.5592103004455566, |
| "rewards/margins": 3.5806994438171387, |
| "rewards/rejected": -3.021489143371582, |
| "step": 260 |
| }, |
| { |
| "epoch": 1.2, |
| "eval_logits/chosen": -1.1710869073867798, |
| "eval_logits/rejected": -1.179579496383667, |
| "eval_logps/chosen": -22.368024826049805, |
| "eval_logps/rejected": -31.889461517333984, |
| "eval_loss": 0.24438533186912537, |
| "eval_rewards/accuracies": 0.8317972421646118, |
| "eval_rewards/chosen": 0.33933624625205994, |
| "eval_rewards/margins": 3.1036696434020996, |
| "eval_rewards/rejected": -2.764333963394165, |
| "eval_runtime": 216.3298, |
| "eval_samples_per_second": 8.016, |
| "eval_steps_per_second": 2.006, |
| "step": 260 |
| }, |
| { |
| "epoch": 1.209230769230769, |
| "grad_norm": 16.513762580218792, |
| "learning_rate": 2.0172385088197803e-07, |
| "logits/chosen": -1.14779531955719, |
| "logits/rejected": -1.1652312278747559, |
| "logps/chosen": -26.26132583618164, |
| "logps/rejected": -40.5022087097168, |
| "loss": 0.2143, |
| "rewards/accuracies": 0.7777777910232544, |
| "rewards/chosen": 0.41341039538383484, |
| "rewards/margins": 3.354189157485962, |
| "rewards/rejected": -2.940778970718384, |
| "step": 262 |
| }, |
| { |
| "epoch": 1.2184615384615385, |
| "grad_norm": 11.212524578416895, |
| "learning_rate": 1.977581092421812e-07, |
| "logits/chosen": -1.1520088911056519, |
| "logits/rejected": -1.1642160415649414, |
| "logps/chosen": -20.592201232910156, |
| "logps/rejected": -30.868377685546875, |
| "loss": 0.1657, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.40944963693618774, |
| "rewards/margins": 3.2444136142730713, |
| "rewards/rejected": -2.8349640369415283, |
| "step": 264 |
| }, |
| { |
| "epoch": 1.2276923076923076, |
| "grad_norm": 11.01146404378747, |
| "learning_rate": 1.9380606713281772e-07, |
| "logits/chosen": -1.1583861112594604, |
| "logits/rejected": -1.1652624607086182, |
| "logps/chosen": -18.12959098815918, |
| "logps/rejected": -34.5963134765625, |
| "loss": 0.2062, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.3663688898086548, |
| "rewards/margins": 3.504619836807251, |
| "rewards/rejected": -3.1382510662078857, |
| "step": 266 |
| }, |
| { |
| "epoch": 1.236923076923077, |
| "grad_norm": 12.264405123220332, |
| "learning_rate": 1.8986876090843664e-07, |
| "logits/chosen": -1.13167142868042, |
| "logits/rejected": -1.14499831199646, |
| "logps/chosen": -20.43359375, |
| "logps/rejected": -37.75240707397461, |
| "loss": 0.1807, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.36272215843200684, |
| "rewards/margins": 3.8877878189086914, |
| "rewards/rejected": -3.5250654220581055, |
| "step": 268 |
| }, |
| { |
| "epoch": 1.2461538461538462, |
| "grad_norm": 11.919291580876626, |
| "learning_rate": 1.859472230593569e-07, |
| "logits/chosen": -1.1225872039794922, |
| "logits/rejected": -1.1367418766021729, |
| "logps/chosen": -26.361604690551758, |
| "logps/rejected": -43.534812927246094, |
| "loss": 0.2145, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.43198204040527344, |
| "rewards/margins": 3.9310781955718994, |
| "rewards/rejected": -3.499096155166626, |
| "step": 270 |
| }, |
| { |
| "epoch": 1.2553846153846153, |
| "grad_norm": 12.440022575260326, |
| "learning_rate": 1.8204248194091425e-07, |
| "logits/chosen": -1.1526453495025635, |
| "logits/rejected": -1.1696141958236694, |
| "logps/chosen": -23.60825538635254, |
| "logps/rejected": -57.63713836669922, |
| "loss": 0.1955, |
| "rewards/accuracies": 0.9305555820465088, |
| "rewards/chosen": 0.27740761637687683, |
| "rewards/margins": 4.90004301071167, |
| "rewards/rejected": -4.622635841369629, |
| "step": 272 |
| }, |
| { |
| "epoch": 1.2646153846153847, |
| "grad_norm": 8.286919730890018, |
| "learning_rate": 1.7815556150379296e-07, |
| "logits/chosen": -1.1683982610702515, |
| "logits/rejected": -1.169435977935791, |
| "logps/chosen": -22.41632652282715, |
| "logps/rejected": -32.76851272583008, |
| "loss": 0.1885, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.5325616002082825, |
| "rewards/margins": 3.4823427200317383, |
| "rewards/rejected": -2.9497809410095215, |
| "step": 274 |
| }, |
| { |
| "epoch": 1.2738461538461539, |
| "grad_norm": 11.685150583165354, |
| "learning_rate": 1.7428748102551234e-07, |
| "logits/chosen": -1.106712818145752, |
| "logits/rejected": -1.1161227226257324, |
| "logps/chosen": -20.291996002197266, |
| "logps/rejected": -28.43364715576172, |
| "loss": 0.1994, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.5047957897186279, |
| "rewards/margins": 3.1466941833496094, |
| "rewards/rejected": -2.6418981552124023, |
| "step": 276 |
| }, |
| { |
| "epoch": 1.283076923076923, |
| "grad_norm": 13.842054601252082, |
| "learning_rate": 1.704392548431391e-07, |
| "logits/chosen": -1.1573395729064941, |
| "logits/rejected": -1.1763123273849487, |
| "logps/chosen": -13.727288246154785, |
| "logps/rejected": -40.552120208740234, |
| "loss": 0.1992, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.0833960473537445, |
| "rewards/margins": 3.4928784370422363, |
| "rewards/rejected": -3.40948224067688, |
| "step": 278 |
| }, |
| { |
| "epoch": 1.2923076923076924, |
| "grad_norm": 19.81840697060037, |
| "learning_rate": 1.6661189208729489e-07, |
| "logits/chosen": -1.1369847059249878, |
| "logits/rejected": -1.1503101587295532, |
| "logps/chosen": -29.371524810791016, |
| "logps/rejected": -31.74928092956543, |
| "loss": 0.174, |
| "rewards/accuracies": 0.9166666865348816, |
| "rewards/chosen": 0.46892601251602173, |
| "rewards/margins": 3.2968459129333496, |
| "rewards/rejected": -2.8279199600219727, |
| "step": 280 |
| }, |
| { |
| "epoch": 1.2923076923076924, |
| "eval_logits/chosen": -1.165863275527954, |
| "eval_logits/rejected": -1.1743441820144653, |
| "eval_logps/chosen": -22.31157875061035, |
| "eval_logps/rejected": -31.91876792907715, |
| "eval_loss": 0.23967565596103668, |
| "eval_rewards/accuracies": 0.8341013789176941, |
| "eval_rewards/chosen": 0.3675578236579895, |
| "eval_rewards/margins": 3.146545171737671, |
| "eval_rewards/rejected": -2.778987407684326, |
| "eval_runtime": 216.3352, |
| "eval_samples_per_second": 8.015, |
| "eval_steps_per_second": 2.006, |
| "step": 280 |
| }, |
| { |
| "epoch": 1.3015384615384615, |
| "grad_norm": 8.930251698810418, |
| "learning_rate": 1.6280639641752942e-07, |
| "logits/chosen": -1.1316086053848267, |
| "logits/rejected": -1.1440240144729614, |
| "logps/chosen": -20.34646987915039, |
| "logps/rejected": -49.82673645019531, |
| "loss": 0.1765, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.23807168006896973, |
| "rewards/margins": 4.113887310028076, |
| "rewards/rejected": -3.8758151531219482, |
| "step": 282 |
| }, |
| { |
| "epoch": 1.3107692307692307, |
| "grad_norm": 12.563220339411409, |
| "learning_rate": 1.5902376575912814e-07, |
| "logits/chosen": -1.11788809299469, |
| "logits/rejected": -1.1216245889663696, |
| "logps/chosen": -26.72078514099121, |
| "logps/rejected": -35.561317443847656, |
| "loss": 0.1887, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.3794720470905304, |
| "rewards/margins": 3.400892734527588, |
| "rewards/rejected": -3.021420478820801, |
| "step": 284 |
| }, |
| { |
| "epoch": 1.32, |
| "grad_norm": 12.663334489473607, |
| "learning_rate": 1.552649920414233e-07, |
| "logits/chosen": -1.1346993446350098, |
| "logits/rejected": -1.135698676109314, |
| "logps/chosen": -30.942975997924805, |
| "logps/rejected": -28.223663330078125, |
| "loss": 0.209, |
| "rewards/accuracies": 0.8055555820465088, |
| "rewards/chosen": 0.1739700883626938, |
| "rewards/margins": 2.763653039932251, |
| "rewards/rejected": -2.5896828174591064, |
| "step": 286 |
| }, |
| { |
| "epoch": 1.3292307692307692, |
| "grad_norm": 14.8989835155845, |
| "learning_rate": 1.5153106093767825e-07, |
| "logits/chosen": -1.0928491353988647, |
| "logits/rejected": -1.115010142326355, |
| "logps/chosen": -18.197795867919922, |
| "logps/rejected": -37.05016326904297, |
| "loss": 0.2571, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.4650332033634186, |
| "rewards/margins": 2.95278000831604, |
| "rewards/rejected": -2.4877467155456543, |
| "step": 288 |
| }, |
| { |
| "epoch": 1.3384615384615386, |
| "grad_norm": 7.959815386261902, |
| "learning_rate": 1.47822951606611e-07, |
| "logits/chosen": -1.1016626358032227, |
| "logits/rejected": -1.1072629690170288, |
| "logps/chosen": -27.025487899780273, |
| "logps/rejected": -32.04999923706055, |
| "loss": 0.1876, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.37108778953552246, |
| "rewards/margins": 3.5628809928894043, |
| "rewards/rejected": -3.191793441772461, |
| "step": 290 |
| }, |
| { |
| "epoch": 1.3476923076923077, |
| "grad_norm": 9.883542506968235, |
| "learning_rate": 1.4414163643562753e-07, |
| "logits/chosen": -1.1510549783706665, |
| "logits/rejected": -1.161637783050537, |
| "logps/chosen": -26.81183433532715, |
| "logps/rejected": -45.584022521972656, |
| "loss": 0.1694, |
| "rewards/accuracies": 0.9166666865348816, |
| "rewards/chosen": 0.7165854573249817, |
| "rewards/margins": 4.145462989807129, |
| "rewards/rejected": -3.428877353668213, |
| "step": 292 |
| }, |
| { |
| "epoch": 1.356923076923077, |
| "grad_norm": 16.819884237605038, |
| "learning_rate": 1.4048808078582942e-07, |
| "logits/chosen": -1.156364917755127, |
| "logits/rejected": -1.158648133277893, |
| "logps/chosen": -25.07522964477539, |
| "logps/rejected": -37.01847839355469, |
| "loss": 0.1916, |
| "rewards/accuracies": 0.9027777910232544, |
| "rewards/chosen": -0.062492769211530685, |
| "rewards/margins": 3.447725534439087, |
| "rewards/rejected": -3.5102179050445557, |
| "step": 294 |
| }, |
| { |
| "epoch": 1.3661538461538463, |
| "grad_norm": 9.730872259730013, |
| "learning_rate": 1.3686324273886528e-07, |
| "logits/chosen": -1.0902260541915894, |
| "logits/rejected": -1.1149543523788452, |
| "logps/chosen": -21.78764533996582, |
| "logps/rejected": -47.82768249511719, |
| "loss": 0.1618, |
| "rewards/accuracies": 0.9444444179534912, |
| "rewards/chosen": 0.330030232667923, |
| "rewards/margins": 4.0784478187561035, |
| "rewards/rejected": -3.748418092727661, |
| "step": 296 |
| }, |
| { |
| "epoch": 1.3753846153846154, |
| "grad_norm": 11.017633003526004, |
| "learning_rate": 1.3326807284568984e-07, |
| "logits/chosen": -1.1744215488433838, |
| "logits/rejected": -1.1781681776046753, |
| "logps/chosen": -20.410446166992188, |
| "logps/rejected": -33.22405242919922, |
| "loss": 0.2013, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.266373872756958, |
| "rewards/margins": 3.171236515045166, |
| "rewards/rejected": -2.904862642288208, |
| "step": 298 |
| }, |
| { |
| "epoch": 1.3846153846153846, |
| "grad_norm": 12.616723945362331, |
| "learning_rate": 1.2970351387729872e-07, |
| "logits/chosen": -1.1809624433517456, |
| "logits/rejected": -1.1951857805252075, |
| "logps/chosen": -18.240955352783203, |
| "logps/rejected": -40.42936706542969, |
| "loss": 0.2077, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.6317293643951416, |
| "rewards/margins": 3.926286458969116, |
| "rewards/rejected": -3.2945568561553955, |
| "step": 300 |
| }, |
| { |
| "epoch": 1.3846153846153846, |
| "eval_logits/chosen": -1.1625326871871948, |
| "eval_logits/rejected": -1.1709260940551758, |
| "eval_logps/chosen": -22.30373764038086, |
| "eval_logps/rejected": -32.03895568847656, |
| "eval_loss": 0.23691046237945557, |
| "eval_rewards/accuracies": 0.8387096524238586, |
| "eval_rewards/chosen": 0.3714797794818878, |
| "eval_rewards/margins": 3.2105631828308105, |
| "eval_rewards/rejected": -2.839083194732666, |
| "eval_runtime": 216.5842, |
| "eval_samples_per_second": 8.006, |
| "eval_steps_per_second": 2.004, |
| "step": 300 |
| }, |
| { |
| "epoch": 1.393846153846154, |
| "grad_norm": 11.126146094324666, |
| "learning_rate": 1.261705005775032e-07, |
| "logits/chosen": -1.1696714162826538, |
| "logits/rejected": -1.1861652135849, |
| "logps/chosen": -22.42890167236328, |
| "logps/rejected": -34.44594192504883, |
| "loss": 0.1635, |
| "rewards/accuracies": 0.9305555820465088, |
| "rewards/chosen": 0.2834773361682892, |
| "rewards/margins": 3.5443296432495117, |
| "rewards/rejected": -3.260852813720703, |
| "step": 302 |
| }, |
| { |
| "epoch": 1.403076923076923, |
| "grad_norm": 10.479052450533084, |
| "learning_rate": 1.2266995941780933e-07, |
| "logits/chosen": -1.130216121673584, |
| "logits/rejected": -1.1414945125579834, |
| "logps/chosen": -25.476299285888672, |
| "logps/rejected": -40.09599304199219, |
| "loss": 0.1598, |
| "rewards/accuracies": 0.9166666865348816, |
| "rewards/chosen": 0.3959537744522095, |
| "rewards/margins": 3.8914499282836914, |
| "rewards/rejected": -3.4954960346221924, |
| "step": 304 |
| }, |
| { |
| "epoch": 1.4123076923076923, |
| "grad_norm": 15.900407241334104, |
| "learning_rate": 1.1920280835446748e-07, |
| "logits/chosen": -1.1561819314956665, |
| "logits/rejected": -1.160946011543274, |
| "logps/chosen": -26.870162963867188, |
| "logps/rejected": -45.102787017822266, |
| "loss": 0.1771, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.5023772120475769, |
| "rewards/margins": 4.30380392074585, |
| "rewards/rejected": -3.801426887512207, |
| "step": 306 |
| }, |
| { |
| "epoch": 1.4215384615384616, |
| "grad_norm": 10.845292151115956, |
| "learning_rate": 1.1576995658775404e-07, |
| "logits/chosen": -1.1523799896240234, |
| "logits/rejected": -1.1634249687194824, |
| "logps/chosen": -20.11031723022461, |
| "logps/rejected": -28.449501037597656, |
| "loss": 0.155, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.4131190776824951, |
| "rewards/margins": 3.466240882873535, |
| "rewards/rejected": -3.053121328353882, |
| "step": 308 |
| }, |
| { |
| "epoch": 1.4307692307692308, |
| "grad_norm": 13.811097447536184, |
| "learning_rate": 1.123723043235491e-07, |
| "logits/chosen": -1.1037707328796387, |
| "logits/rejected": -1.1196866035461426, |
| "logps/chosen": -22.25092315673828, |
| "logps/rejected": -41.13553237915039, |
| "loss": 0.2394, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.44290411472320557, |
| "rewards/margins": 3.9364805221557617, |
| "rewards/rejected": -3.4935765266418457, |
| "step": 310 |
| }, |
| { |
| "epoch": 1.44, |
| "grad_norm": 7.336736527232887, |
| "learning_rate": 1.0901074253727336e-07, |
| "logits/chosen": -1.132401943206787, |
| "logits/rejected": -1.1375315189361572, |
| "logps/chosen": -21.84718132019043, |
| "logps/rejected": -32.056617736816406, |
| "loss": 0.1639, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.543586015701294, |
| "rewards/margins": 3.536188840866089, |
| "rewards/rejected": -2.9926023483276367, |
| "step": 312 |
| }, |
| { |
| "epoch": 1.4492307692307693, |
| "grad_norm": 9.238298739154985, |
| "learning_rate": 1.056861527402452e-07, |
| "logits/chosen": -1.1301486492156982, |
| "logits/rejected": -1.130847454071045, |
| "logps/chosen": -30.35249137878418, |
| "logps/rejected": -39.42829513549805, |
| "loss": 0.1854, |
| "rewards/accuracies": 0.8055555820465088, |
| "rewards/chosen": 0.6695830821990967, |
| "rewards/margins": 3.61427903175354, |
| "rewards/rejected": -2.9446957111358643, |
| "step": 314 |
| }, |
| { |
| "epoch": 1.4584615384615385, |
| "grad_norm": 13.901867549459764, |
| "learning_rate": 1.0239940674851941e-07, |
| "logits/chosen": -1.1156858205795288, |
| "logits/rejected": -1.114392638206482, |
| "logps/chosen": -24.01244354248047, |
| "logps/rejected": -34.20494842529297, |
| "loss": 0.1866, |
| "rewards/accuracies": 0.9027777910232544, |
| "rewards/chosen": 0.37583643198013306, |
| "rewards/margins": 3.5291662216186523, |
| "rewards/rejected": -3.153329610824585, |
| "step": 316 |
| }, |
| { |
| "epoch": 1.4676923076923076, |
| "grad_norm": 11.080424296345777, |
| "learning_rate": 9.915136645426883e-08, |
| "logits/chosen": -1.1818937063217163, |
| "logits/rejected": -1.1808428764343262, |
| "logps/chosen": -24.881999969482422, |
| "logps/rejected": -28.97332763671875, |
| "loss": 0.173, |
| "rewards/accuracies": 0.9305555820465088, |
| "rewards/chosen": 0.36953669786453247, |
| "rewards/margins": 3.261909246444702, |
| "rewards/rejected": -2.8923726081848145, |
| "step": 318 |
| }, |
| { |
| "epoch": 1.476923076923077, |
| "grad_norm": 15.189646270302608, |
| "learning_rate": 9.594288359976815e-08, |
| "logits/chosen": -1.1282167434692383, |
| "logits/rejected": -1.1426851749420166, |
| "logps/chosen": -17.99266815185547, |
| "logps/rejected": -47.12626266479492, |
| "loss": 0.2092, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.30799973011016846, |
| "rewards/margins": 4.037694454193115, |
| "rewards/rejected": -3.729694366455078, |
| "step": 320 |
| }, |
| { |
| "epoch": 1.476923076923077, |
| "eval_logits/chosen": -1.1610218286514282, |
| "eval_logits/rejected": -1.1692686080932617, |
| "eval_logps/chosen": -22.297130584716797, |
| "eval_logps/rejected": -32.10142135620117, |
| "eval_loss": 0.23491987586021423, |
| "eval_rewards/accuracies": 0.8329492807388306, |
| "eval_rewards/chosen": 0.3747842013835907, |
| "eval_rewards/margins": 3.245098829269409, |
| "eval_rewards/rejected": -2.870314836502075, |
| "eval_runtime": 216.0919, |
| "eval_samples_per_second": 8.024, |
| "eval_steps_per_second": 2.008, |
| "step": 320 |
| }, |
| { |
| "epoch": 1.4861538461538462, |
| "grad_norm": 11.193355120949441, |
| "learning_rate": 9.277479955403886e-08, |
| "logits/chosen": -1.147449016571045, |
| "logits/rejected": -1.1808828115463257, |
| "logps/chosen": -19.78190040588379, |
| "logps/rejected": -68.74774932861328, |
| "loss": 0.1519, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.25842922925949097, |
| "rewards/margins": 5.480890274047852, |
| "rewards/rejected": -5.222461223602295, |
| "step": 322 |
| }, |
| { |
| "epoch": 1.4953846153846153, |
| "grad_norm": 11.257040825977688, |
| "learning_rate": 8.964794509221507e-08, |
| "logits/chosen": -1.1383910179138184, |
| "logits/rejected": -1.148794412612915, |
| "logps/chosen": -25.653322219848633, |
| "logps/rejected": -34.04636001586914, |
| "loss": 0.1653, |
| "rewards/accuracies": 0.9305555820465088, |
| "rewards/chosen": 0.33952367305755615, |
| "rewards/margins": 3.5638911724090576, |
| "rewards/rejected": -3.224367380142212, |
| "step": 324 |
| }, |
| { |
| "epoch": 1.5046153846153847, |
| "grad_norm": 14.248331413419937, |
| "learning_rate": 8.656314017768693e-08, |
| "logits/chosen": -1.1353636980056763, |
| "logits/rejected": -1.1488914489746094, |
| "logps/chosen": -23.45088768005371, |
| "logps/rejected": -36.34320831298828, |
| "loss": 0.19, |
| "rewards/accuracies": 0.9027777910232544, |
| "rewards/chosen": 0.5625240802764893, |
| "rewards/margins": 3.636873483657837, |
| "rewards/rejected": -3.0743494033813477, |
| "step": 326 |
| }, |
| { |
| "epoch": 1.5138461538461538, |
| "grad_norm": 11.13430757826836, |
| "learning_rate": 8.352119374707977e-08, |
| "logits/chosen": -1.1736154556274414, |
| "logits/rejected": -1.1819250583648682, |
| "logps/chosen": -21.08655548095703, |
| "logps/rejected": -31.81151580810547, |
| "loss": 0.1618, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.3814205825328827, |
| "rewards/margins": 3.455685615539551, |
| "rewards/rejected": -3.0742650032043457, |
| "step": 328 |
| }, |
| { |
| "epoch": 1.523076923076923, |
| "grad_norm": 9.775792350949882, |
| "learning_rate": 8.052290349812419e-08, |
| "logits/chosen": -1.1424063444137573, |
| "logits/rejected": -1.1474817991256714, |
| "logps/chosen": -21.133007049560547, |
| "logps/rejected": -25.102752685546875, |
| "loss": 0.2071, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.4940270781517029, |
| "rewards/margins": 2.9714784622192383, |
| "rewards/rejected": -2.4774513244628906, |
| "step": 330 |
| }, |
| { |
| "epoch": 1.5323076923076924, |
| "grad_norm": 6.768309866947245, |
| "learning_rate": 7.756905568047392e-08, |
| "logits/chosen": -1.1152650117874146, |
| "logits/rejected": -1.12236750125885, |
| "logps/chosen": -17.50248146057129, |
| "logps/rejected": -29.518686294555664, |
| "loss": 0.159, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.6183215379714966, |
| "rewards/margins": 3.7438418865203857, |
| "rewards/rejected": -3.1255204677581787, |
| "step": 332 |
| }, |
| { |
| "epoch": 1.5415384615384615, |
| "grad_norm": 12.853827774295516, |
| "learning_rate": 7.46604248895252e-08, |
| "logits/chosen": -1.1082737445831299, |
| "logits/rejected": -1.1175150871276855, |
| "logps/chosen": -20.219505310058594, |
| "logps/rejected": -28.43560218811035, |
| "loss": 0.1827, |
| "rewards/accuracies": 0.9027777910232544, |
| "rewards/chosen": 0.34581294655799866, |
| "rewards/margins": 3.1769955158233643, |
| "rewards/rejected": -2.8311829566955566, |
| "step": 334 |
| }, |
| { |
| "epoch": 1.5507692307692307, |
| "grad_norm": 7.493668682648857, |
| "learning_rate": 7.179777386329275e-08, |
| "logits/chosen": -1.1045269966125488, |
| "logits/rejected": -1.1183186769485474, |
| "logps/chosen": -21.421226501464844, |
| "logps/rejected": -39.41886901855469, |
| "loss": 0.1748, |
| "rewards/accuracies": 0.9027777910232544, |
| "rewards/chosen": 0.5396389365196228, |
| "rewards/margins": 3.9202401638031006, |
| "rewards/rejected": -3.380601167678833, |
| "step": 336 |
| }, |
| { |
| "epoch": 1.56, |
| "grad_norm": 12.452229910069226, |
| "learning_rate": 6.898185328239467e-08, |
| "logits/chosen": -1.145583987236023, |
| "logits/rejected": -1.1488795280456543, |
| "logps/chosen": -22.65854263305664, |
| "logps/rejected": -31.751142501831055, |
| "loss": 0.1845, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.2917179465293884, |
| "rewards/margins": 3.111690044403076, |
| "rewards/rejected": -2.819972038269043, |
| "step": 338 |
| }, |
| { |
| "epoch": 1.5692307692307692, |
| "grad_norm": 10.84177308211244, |
| "learning_rate": 6.621340157319996e-08, |
| "logits/chosen": -1.1560921669006348, |
| "logits/rejected": -1.1605477333068848, |
| "logps/chosen": -16.325712203979492, |
| "logps/rejected": -24.499792098999023, |
| "loss": 0.2045, |
| "rewards/accuracies": 0.9027777910232544, |
| "rewards/chosen": 0.44531428813934326, |
| "rewards/margins": 3.1462950706481934, |
| "rewards/rejected": -2.7009804248809814, |
| "step": 340 |
| }, |
| { |
| "epoch": 1.5692307692307692, |
| "eval_logits/chosen": -1.1584707498550415, |
| "eval_logits/rejected": -1.1668710708618164, |
| "eval_logps/chosen": -22.341110229492188, |
| "eval_logps/rejected": -32.223533630371094, |
| "eval_loss": 0.23495733737945557, |
| "eval_rewards/accuracies": 0.8341013789176941, |
| "eval_rewards/chosen": 0.35279345512390137, |
| "eval_rewards/margins": 3.2841641902923584, |
| "eval_rewards/rejected": -2.931370496749878, |
| "eval_runtime": 216.2511, |
| "eval_samples_per_second": 8.018, |
| "eval_steps_per_second": 2.007, |
| "step": 340 |
| }, |
| { |
| "epoch": 1.5784615384615384, |
| "grad_norm": 8.225696594197464, |
| "learning_rate": 6.349314471418849e-08, |
| "logits/chosen": -1.0857443809509277, |
| "logits/rejected": -1.0922576189041138, |
| "logps/chosen": -16.084243774414062, |
| "logps/rejected": -30.81378173828125, |
| "loss": 0.1803, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.5106647610664368, |
| "rewards/margins": 3.7973814010620117, |
| "rewards/rejected": -3.2867166996002197, |
| "step": 342 |
| }, |
| { |
| "epoch": 1.5876923076923077, |
| "grad_norm": 15.760247716168218, |
| "learning_rate": 6.082179604557616e-08, |
| "logits/chosen": -1.1193811893463135, |
| "logits/rejected": -1.121721863746643, |
| "logps/chosen": -22.19783592224121, |
| "logps/rejected": -28.761178970336914, |
| "loss": 0.197, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.40734562277793884, |
| "rewards/margins": 3.452158212661743, |
| "rewards/rejected": -3.0448129177093506, |
| "step": 344 |
| }, |
| { |
| "epoch": 1.596923076923077, |
| "grad_norm": 10.909974494088763, |
| "learning_rate": 5.8200056082253453e-08, |
| "logits/chosen": -1.125333547592163, |
| "logits/rejected": -1.142914056777954, |
| "logps/chosen": -19.27569007873535, |
| "logps/rejected": -45.170040130615234, |
| "loss": 0.1653, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.4003957509994507, |
| "rewards/margins": 4.2396368980407715, |
| "rewards/rejected": -3.839240550994873, |
| "step": 346 |
| }, |
| { |
| "epoch": 1.606153846153846, |
| "grad_norm": 10.855639719670084, |
| "learning_rate": 5.5628612330087724e-08, |
| "logits/chosen": -1.131655216217041, |
| "logits/rejected": -1.1401116847991943, |
| "logps/chosen": -17.995466232299805, |
| "logps/rejected": -32.176475524902344, |
| "loss": 0.1826, |
| "rewards/accuracies": 0.8472222089767456, |
| "rewards/chosen": 0.4925755262374878, |
| "rewards/margins": 3.6894967555999756, |
| "rewards/rejected": -3.196920871734619, |
| "step": 348 |
| }, |
| { |
| "epoch": 1.6153846153846154, |
| "grad_norm": 16.085282454030374, |
| "learning_rate": 5.310813910563644e-08, |
| "logits/chosen": -1.0810273885726929, |
| "logits/rejected": -1.0798935890197754, |
| "logps/chosen": -22.392784118652344, |
| "logps/rejected": -28.961748123168945, |
| "loss": 0.2082, |
| "rewards/accuracies": 0.7916666865348816, |
| "rewards/chosen": 0.39071983098983765, |
| "rewards/margins": 2.970240592956543, |
| "rewards/rejected": -2.5795204639434814, |
| "step": 350 |
| }, |
| { |
| "epoch": 1.6246153846153846, |
| "grad_norm": 16.9671493136513, |
| "learning_rate": 5.0639297359319846e-08, |
| "logits/chosen": -1.1683417558670044, |
| "logits/rejected": -1.1672459840774536, |
| "logps/chosen": -24.353551864624023, |
| "logps/rejected": -27.454164505004883, |
| "loss": 0.2106, |
| "rewards/accuracies": 0.8333333134651184, |
| "rewards/chosen": 0.26455923914909363, |
| "rewards/margins": 2.982168674468994, |
| "rewards/rejected": -2.717609167098999, |
| "step": 352 |
| }, |
| { |
| "epoch": 1.6338461538461537, |
| "grad_norm": 10.455898381248911, |
| "learning_rate": 4.8222734502097655e-08, |
| "logits/chosen": -1.1433789730072021, |
| "logits/rejected": -1.153548240661621, |
| "logps/chosen": -24.5914363861084, |
| "logps/rejected": -42.36714172363281, |
| "loss": 0.1885, |
| "rewards/accuracies": 0.8194444179534912, |
| "rewards/chosen": 0.36157724261283875, |
| "rewards/margins": 3.6608800888061523, |
| "rewards/rejected": -3.2993030548095703, |
| "step": 354 |
| }, |
| { |
| "epoch": 1.643076923076923, |
| "grad_norm": 19.280259828969186, |
| "learning_rate": 4.5859084235697235e-08, |
| "logits/chosen": -1.164656639099121, |
| "logits/rejected": -1.1599383354187012, |
| "logps/chosen": -19.223194122314453, |
| "logps/rejected": -24.446197509765625, |
| "loss": 0.2371, |
| "rewards/accuracies": 0.8611111044883728, |
| "rewards/chosen": 0.3862743377685547, |
| "rewards/margins": 2.9600579738616943, |
| "rewards/rejected": -2.5737838745117188, |
| "step": 356 |
| }, |
| { |
| "epoch": 1.6523076923076923, |
| "grad_norm": 8.14493222848995, |
| "learning_rate": 4.35489663864359e-08, |
| "logits/chosen": -1.0972024202346802, |
| "logits/rejected": -1.1305886507034302, |
| "logps/chosen": -17.79538345336914, |
| "logps/rejected": -59.57120895385742, |
| "loss": 0.2046, |
| "rewards/accuracies": 0.8888888955116272, |
| "rewards/chosen": 0.5223473310470581, |
| "rewards/margins": 4.91096830368042, |
| "rewards/rejected": -4.388620853424072, |
| "step": 358 |
| }, |
| { |
| "epoch": 1.6615384615384614, |
| "grad_norm": 11.376614389062514, |
| "learning_rate": 4.1292986742682254e-08, |
| "logits/chosen": -1.140592098236084, |
| "logits/rejected": -1.1457772254943848, |
| "logps/chosen": -19.596229553222656, |
| "logps/rejected": -32.57119369506836, |
| "loss": 0.1368, |
| "rewards/accuracies": 0.9166666865348816, |
| "rewards/chosen": 0.34850603342056274, |
| "rewards/margins": 3.6875181198120117, |
| "rewards/rejected": -3.339012622833252, |
| "step": 360 |
| }, |
| { |
| "epoch": 1.6615384615384614, |
| "eval_logits/chosen": -1.1585197448730469, |
| "eval_logits/rejected": -1.1669610738754272, |
| "eval_logps/chosen": -22.363513946533203, |
| "eval_logps/rejected": -32.30293273925781, |
| "eval_loss": 0.23404575884342194, |
| "eval_rewards/accuracies": 0.8352534770965576, |
| "eval_rewards/chosen": 0.3415912091732025, |
| "eval_rewards/margins": 3.3126602172851562, |
| "eval_rewards/rejected": -2.9710693359375, |
| "eval_runtime": 216.0202, |
| "eval_samples_per_second": 8.027, |
| "eval_steps_per_second": 2.009, |
| "step": 360 |
| } |
| ], |
| "logging_steps": 2, |
| "max_steps": 432, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 20, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
