| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.27938535222510474, | |
| "eval_steps": 500, | |
| "global_step": 200, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6335.173723493303, | |
| "epoch": 0.0013969267611255239, | |
| "grad_norm": 0.1530565852782136, | |
| "learning_rate": 0.0, | |
| "loss": 0.0119, | |
| "num_tokens": 1330201.0, | |
| "reward": 0.3924093656241894, | |
| "reward_std": 0.32559600685323986, | |
| "rewards/reward_func": 0.3924093528517655, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "epoch": 0.0027938535222510478, | |
| "grad_norm": 0.15303408357738274, | |
| "learning_rate": 1.3888888888888887e-08, | |
| "loss": 0.0119, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.010408175710056509, | |
| "epoch": 0.004190780283376571, | |
| "grad_norm": 0.1764633199425111, | |
| "learning_rate": 2.7777777777777774e-08, | |
| "loss": 0.0125, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.010680466823812042, | |
| "epoch": 0.0055877070445020955, | |
| "grad_norm": 0.2275277428050235, | |
| "learning_rate": 4.166666666666666e-08, | |
| "loss": 0.0126, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5737.668526785715, | |
| "epoch": 0.006984633805627619, | |
| "grad_norm": 0.19107039038527468, | |
| "learning_rate": 5.555555555555555e-08, | |
| "loss": 0.0293, | |
| "num_tokens": 2545372.0, | |
| "reward": 0.4294457712343761, | |
| "reward_std": 0.37008823986564365, | |
| "rewards/reward_func": 0.4294457531401089, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.010828850524766105, | |
| "epoch": 0.008381560566753142, | |
| "grad_norm": 0.2240354781456917, | |
| "learning_rate": 6.944444444444444e-08, | |
| "loss": 0.0301, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.010841753533376115, | |
| "epoch": 0.009778487327878668, | |
| "grad_norm": 0.2767916742737982, | |
| "learning_rate": 8.333333333333333e-08, | |
| "loss": 0.0301, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.010758176312914916, | |
| "epoch": 0.011175414089004191, | |
| "grad_norm": 0.2668705057271519, | |
| "learning_rate": 9.722222222222222e-08, | |
| "loss": 0.0301, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6511.857439313616, | |
| "epoch": 0.012572340850129714, | |
| "grad_norm": 0.19266419886931196, | |
| "learning_rate": 1.111111111111111e-07, | |
| "loss": 0.0051, | |
| "num_tokens": 3911895.0, | |
| "reward": 0.32038674876093864, | |
| "reward_std": 0.3298808889729636, | |
| "rewards/reward_func": 0.32038673226322445, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.012134518141725234, | |
| "epoch": 0.013969267611255238, | |
| "grad_norm": 0.21591989333017872, | |
| "learning_rate": 1.25e-07, | |
| "loss": 0.0058, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.012228245115173715, | |
| "epoch": 0.015366194372380763, | |
| "grad_norm": 0.20676433229073893, | |
| "learning_rate": 1.3888888888888888e-07, | |
| "loss": 0.0058, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.012062976469418831, | |
| "epoch": 0.016763121133506285, | |
| "grad_norm": 0.2100849809088676, | |
| "learning_rate": 1.527777777777778e-07, | |
| "loss": 0.0058, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5776.398141043527, | |
| "epoch": 0.01816004789463181, | |
| "grad_norm": 0.21639349074679098, | |
| "learning_rate": 1.6666666666666665e-07, | |
| "loss": 0.0233, | |
| "num_tokens": 5132268.0, | |
| "reward": 0.4814984883580889, | |
| "reward_std": 0.3754643425345421, | |
| "rewards/reward_func": 0.48149845749139786, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.010207080947501319, | |
| "epoch": 0.019556974655757335, | |
| "grad_norm": 0.22611722201173162, | |
| "learning_rate": 1.8055555555555554e-07, | |
| "loss": 0.0241, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.010003214618856353, | |
| "epoch": 0.02095390141688286, | |
| "grad_norm": 0.21387408972131952, | |
| "learning_rate": 1.9444444444444445e-07, | |
| "loss": 0.024, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.010145484004169703, | |
| "epoch": 0.022350828178008382, | |
| "grad_norm": 0.2965931721160837, | |
| "learning_rate": 2.0833333333333333e-07, | |
| "loss": 0.024, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6396.923740931919, | |
| "epoch": 0.023747754939133905, | |
| "grad_norm": 0.17812701051388663, | |
| "learning_rate": 2.222222222222222e-07, | |
| "loss": 0.0143, | |
| "num_tokens": 6474962.0, | |
| "reward": 0.4179248466555561, | |
| "reward_std": 0.3605238295027188, | |
| "rewards/reward_func": 0.41792482949261156, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.010267588176897593, | |
| "epoch": 0.02514468170025943, | |
| "grad_norm": 0.18587492539762027, | |
| "learning_rate": 2.361111111111111e-07, | |
| "loss": 0.015, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.010148471055020179, | |
| "epoch": 0.026541608461384952, | |
| "grad_norm": 0.1810944495339226, | |
| "learning_rate": 2.5e-07, | |
| "loss": 0.0149, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.01020466532957341, | |
| "epoch": 0.027938535222510476, | |
| "grad_norm": 0.1924193708866086, | |
| "learning_rate": 2.638888888888889e-07, | |
| "loss": 0.015, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5085.755362374442, | |
| "epoch": 0.029335461983636, | |
| "grad_norm": 0.3165497451368565, | |
| "learning_rate": 2.7777777777777776e-07, | |
| "loss": 0.0104, | |
| "num_tokens": 7560523.0, | |
| "reward": 0.464439152606896, | |
| "reward_std": 0.3689843277846064, | |
| "rewards/reward_func": 0.46443913131952286, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.010546388570219278, | |
| "epoch": 0.030732388744761526, | |
| "grad_norm": 0.3395774959095009, | |
| "learning_rate": 2.916666666666667e-07, | |
| "loss": 0.0113, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.010336563357019, | |
| "epoch": 0.032129315505887046, | |
| "grad_norm": 0.31582946500885495, | |
| "learning_rate": 3.055555555555556e-07, | |
| "loss": 0.0112, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.010099471979109305, | |
| "epoch": 0.03352624226701257, | |
| "grad_norm": 0.36247326697578075, | |
| "learning_rate": 3.194444444444444e-07, | |
| "loss": 0.0112, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6461.342215401785, | |
| "epoch": 0.03492316902813809, | |
| "grad_norm": 0.15567466235487185, | |
| "learning_rate": 3.333333333333333e-07, | |
| "loss": 0.0193, | |
| "num_tokens": 8914764.0, | |
| "reward": 0.38339252876383917, | |
| "reward_std": 0.3311073939715113, | |
| "rewards/reward_func": 0.3833925181201526, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.009515652732391442, | |
| "epoch": 0.03632009578926362, | |
| "grad_norm": 0.25805173133838866, | |
| "learning_rate": 3.472222222222222e-07, | |
| "loss": 0.0199, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.009263422406677688, | |
| "epoch": 0.03771702255038915, | |
| "grad_norm": 0.4090162080972072, | |
| "learning_rate": 3.6111111111111107e-07, | |
| "loss": 0.0199, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.009094293162758862, | |
| "epoch": 0.03911394931151467, | |
| "grad_norm": 0.14691760483745145, | |
| "learning_rate": 3.75e-07, | |
| "loss": 0.0198, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5968.250279017857, | |
| "epoch": 0.040510876072640194, | |
| "grad_norm": 0.16406943154337036, | |
| "learning_rate": 3.888888888888889e-07, | |
| "loss": 0.0115, | |
| "num_tokens": 10173741.0, | |
| "reward": 0.3594382884246962, | |
| "reward_std": 0.32283720054796766, | |
| "rewards/reward_func": 0.35943826926606043, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.01090411888435483, | |
| "epoch": 0.04190780283376572, | |
| "grad_norm": 0.21927196182565817, | |
| "learning_rate": 4.027777777777778e-07, | |
| "loss": 0.0124, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.010733458446338773, | |
| "epoch": 0.04330472959489124, | |
| "grad_norm": 0.3447213393816689, | |
| "learning_rate": 4.1666666666666667e-07, | |
| "loss": 0.0121, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.010630057725523199, | |
| "epoch": 0.044701656356016764, | |
| "grad_norm": 0.3620568701207513, | |
| "learning_rate": 4.3055555555555555e-07, | |
| "loss": 0.0121, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5812.648210797991, | |
| "epoch": 0.04609858311714229, | |
| "grad_norm": 0.16987419224516487, | |
| "learning_rate": 4.444444444444444e-07, | |
| "loss": 0.0179, | |
| "num_tokens": 11403427.0, | |
| "reward": 0.3713971259338515, | |
| "reward_std": 0.3195132836699486, | |
| "rewards/reward_func": 0.3713971110326903, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.010423385953929807, | |
| "epoch": 0.04749550987826781, | |
| "grad_norm": 0.16329217057193554, | |
| "learning_rate": 4.5833333333333327e-07, | |
| "loss": 0.0185, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.010347368278806763, | |
| "epoch": 0.048892436639393334, | |
| "grad_norm": 0.15422959333602954, | |
| "learning_rate": 4.722222222222222e-07, | |
| "loss": 0.0185, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.010140369646251202, | |
| "epoch": 0.05028936340051886, | |
| "grad_norm": 0.14689712083262277, | |
| "learning_rate": 4.861111111111111e-07, | |
| "loss": 0.0184, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5546.617606026785, | |
| "epoch": 0.05168629016164438, | |
| "grad_norm": 0.23439115342799705, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0161, | |
| "num_tokens": 12578819.0, | |
| "reward": 0.4297656629766737, | |
| "reward_std": 0.357943703021322, | |
| "rewards/reward_func": 0.42976564009274754, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.010506943932601384, | |
| "epoch": 0.053083216922769905, | |
| "grad_norm": 0.28362735010406454, | |
| "learning_rate": 5.138888888888889e-07, | |
| "loss": 0.0168, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.010338681018246072, | |
| "epoch": 0.05448014368389543, | |
| "grad_norm": 0.22707404637725537, | |
| "learning_rate": 5.277777777777777e-07, | |
| "loss": 0.0167, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.010024449788033962, | |
| "epoch": 0.05587707044502095, | |
| "grad_norm": 0.17811231341381648, | |
| "learning_rate": 5.416666666666666e-07, | |
| "loss": 0.0165, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6501.444231305803, | |
| "epoch": 0.057273997206146475, | |
| "grad_norm": 0.19673275542337104, | |
| "learning_rate": 5.555555555555555e-07, | |
| "loss": 0.0203, | |
| "num_tokens": 13942392.0, | |
| "reward": 0.39034509126629147, | |
| "reward_std": 0.3664030634931156, | |
| "rewards/reward_func": 0.3903450625283377, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.010402361529746227, | |
| "epoch": 0.058670923967272, | |
| "grad_norm": 0.46687244963414065, | |
| "learning_rate": 5.694444444444444e-07, | |
| "loss": 0.0211, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.010496154893189669, | |
| "epoch": 0.06006785072839753, | |
| "grad_norm": 0.19428194405187207, | |
| "learning_rate": 5.833333333333334e-07, | |
| "loss": 0.0209, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.010158670766811286, | |
| "epoch": 0.06146477748952305, | |
| "grad_norm": 0.15306555166745645, | |
| "learning_rate": 5.972222222222222e-07, | |
| "loss": 0.0208, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6369.643048967634, | |
| "epoch": 0.06286170425064858, | |
| "grad_norm": 0.1871015916141642, | |
| "learning_rate": 6.111111111111112e-07, | |
| "loss": 0.0161, | |
| "num_tokens": 15279078.0, | |
| "reward": 0.36025070399045944, | |
| "reward_std": 0.35143060450042996, | |
| "rewards/reward_func": 0.360250677381243, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.01079715442444597, | |
| "epoch": 0.06425863101177409, | |
| "grad_norm": 0.6294866160536174, | |
| "learning_rate": 6.249999999999999e-07, | |
| "loss": 0.0169, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.010736021545848675, | |
| "epoch": 0.06565555777289962, | |
| "grad_norm": 2.137657977104326, | |
| "learning_rate": 6.388888888888888e-07, | |
| "loss": 0.0171, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.010864090407267213, | |
| "epoch": 0.06705248453402514, | |
| "grad_norm": 0.18428434234054034, | |
| "learning_rate": 6.527777777777777e-07, | |
| "loss": 0.0166, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6655.908412388393, | |
| "epoch": 0.06844941129515067, | |
| "grad_norm": 0.16953542603782387, | |
| "learning_rate": 6.666666666666666e-07, | |
| "loss": 0.0148, | |
| "num_tokens": 16672949.0, | |
| "reward": 0.35046685327376637, | |
| "reward_std": 0.30536604832325664, | |
| "rewards/reward_func": 0.3504668373082365, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.010905002310339893, | |
| "epoch": 0.06984633805627619, | |
| "grad_norm": 1.9607105946099048, | |
| "learning_rate": 6.805555555555556e-07, | |
| "loss": 0.0153, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.011137549964977162, | |
| "epoch": 0.07124326481740172, | |
| "grad_norm": 0.16619610980435887, | |
| "learning_rate": 6.944444444444444e-07, | |
| "loss": 0.0153, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.011474365035870246, | |
| "epoch": 0.07264019157852725, | |
| "grad_norm": 0.13731779782413311, | |
| "learning_rate": 7.083333333333334e-07, | |
| "loss": 0.0151, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6259.326799665178, | |
| "epoch": 0.07403711833965276, | |
| "grad_norm": 0.18005417578145988, | |
| "learning_rate": 7.222222222222221e-07, | |
| "loss": 0.0124, | |
| "num_tokens": 17987081.0, | |
| "reward": 0.3450017161667347, | |
| "reward_std": 0.3385175902928625, | |
| "rewards/reward_func": 0.34500169487936155, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.012061000069869416, | |
| "epoch": 0.0754340451007783, | |
| "grad_norm": 0.3653574862365132, | |
| "learning_rate": 7.361111111111111e-07, | |
| "loss": 0.0132, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.01210923127031752, | |
| "epoch": 0.07683097186190381, | |
| "grad_norm": 0.1572526876806962, | |
| "learning_rate": 7.5e-07, | |
| "loss": 0.013, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.012545851857534476, | |
| "epoch": 0.07822789862302934, | |
| "grad_norm": 0.1348151811096135, | |
| "learning_rate": 7.638888888888888e-07, | |
| "loss": 0.0128, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 6945.081944056919, | |
| "epoch": 0.07962482538415486, | |
| "grad_norm": 0.1355243529384343, | |
| "learning_rate": 7.777777777777778e-07, | |
| "loss": 0.0203, | |
| "num_tokens": 19437527.0, | |
| "reward": 0.2853341962077788, | |
| "reward_std": 0.28026825402464184, | |
| "rewards/reward_func": 0.28533417971006464, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.010294904433456915, | |
| "epoch": 0.08102175214528039, | |
| "grad_norm": 1.4215854683138216, | |
| "learning_rate": 7.916666666666666e-07, | |
| "loss": 0.0211, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.010296880932790893, | |
| "epoch": 0.0824186789064059, | |
| "grad_norm": 0.1328390444888187, | |
| "learning_rate": 8.055555555555556e-07, | |
| "loss": 0.0208, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.010402131692639418, | |
| "epoch": 0.08381560566753143, | |
| "grad_norm": 0.10630427559017747, | |
| "learning_rate": 8.194444444444443e-07, | |
| "loss": 0.0207, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5832.984967912947, | |
| "epoch": 0.08521253242865695, | |
| "grad_norm": 0.2568979639198368, | |
| "learning_rate": 8.333333333333333e-07, | |
| "loss": 0.0135, | |
| "num_tokens": 20669861.0, | |
| "reward": 0.37696379157049315, | |
| "reward_std": 0.3524509519338608, | |
| "rewards/reward_func": 0.37696376868656706, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.011475847900978156, | |
| "epoch": 0.08660945918978248, | |
| "grad_norm": 0.26568198717380825, | |
| "learning_rate": 8.472222222222222e-07, | |
| "loss": 0.0142, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.011013164584125792, | |
| "epoch": 0.088006385950908, | |
| "grad_norm": 0.5160311893436309, | |
| "learning_rate": 8.611111111111111e-07, | |
| "loss": 0.014, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.010899153238694583, | |
| "epoch": 0.08940331271203353, | |
| "grad_norm": 0.16319985648001487, | |
| "learning_rate": 8.75e-07, | |
| "loss": 0.0138, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5130.444091796875, | |
| "epoch": 0.09080023947315904, | |
| "grad_norm": 0.28767641683692746, | |
| "learning_rate": 8.888888888888888e-07, | |
| "loss": 0.0025, | |
| "num_tokens": 21764127.0, | |
| "reward": 0.46981193338121685, | |
| "reward_std": 0.3410501836666039, | |
| "rewards/reward_func": 0.4698119152869497, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.011167634411581926, | |
| "epoch": 0.09219716623428457, | |
| "grad_norm": 1786685.0973493713, | |
| "learning_rate": 9.027777777777778e-07, | |
| "loss": 31.3025, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.011178254083331143, | |
| "epoch": 0.09359409299541009, | |
| "grad_norm": 0.2699319084211939, | |
| "learning_rate": 9.166666666666665e-07, | |
| "loss": 0.0029, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.010996093108717884, | |
| "epoch": 0.09499101975653562, | |
| "grad_norm": 0.1796173018225043, | |
| "learning_rate": 9.305555555555555e-07, | |
| "loss": 0.0026, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5374.097150530134, | |
| "epoch": 0.09638794651766115, | |
| "grad_norm": 0.29248413368918635, | |
| "learning_rate": 9.444444444444444e-07, | |
| "loss": 0.0204, | |
| "num_tokens": 22905462.0, | |
| "reward": 0.42480308030332836, | |
| "reward_std": 0.3633535067949976, | |
| "rewards/reward_func": 0.4248030515653746, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.010870669968426228, | |
| "epoch": 0.09778487327878667, | |
| "grad_norm": 0.2916766016433164, | |
| "learning_rate": 9.583333333333334e-07, | |
| "loss": 0.0211, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.010642989072948694, | |
| "epoch": 0.0991818000399122, | |
| "grad_norm": 0.2574669164587784, | |
| "learning_rate": 9.722222222222222e-07, | |
| "loss": 0.0209, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.010603911941871047, | |
| "epoch": 0.10057872680103772, | |
| "grad_norm": 0.3923293286285201, | |
| "learning_rate": 9.861111111111112e-07, | |
| "loss": 0.0206, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 4780.178763253348, | |
| "epoch": 0.10197565356216325, | |
| "grad_norm": 0.3357433852104651, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0111, | |
| "num_tokens": 23928872.0, | |
| "reward": 0.49998574065310614, | |
| "reward_std": 0.3773101898176329, | |
| "rewards/reward_func": 0.4999857119151524, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.012297127628698945, | |
| "epoch": 0.10337258032328876, | |
| "grad_norm": 0.24331462448033314, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0117, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.012708199177203434, | |
| "epoch": 0.10476950708441429, | |
| "grad_norm": 0.6107245136629853, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0114, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.013628917180800013, | |
| "epoch": 0.10616643384553981, | |
| "grad_norm": 0.1571665107730879, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0112, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5568.459463936942, | |
| "epoch": 0.10756336060666534, | |
| "grad_norm": 0.3303048447870775, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0085, | |
| "num_tokens": 25108534.0, | |
| "reward": 0.3940093676958765, | |
| "reward_std": 0.32062976009079386, | |
| "rewards/reward_func": 0.39400935066597803, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.012926211846726281, | |
| "epoch": 0.10896028736779086, | |
| "grad_norm": 0.4586726422820888, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0092, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.013429554013003196, | |
| "epoch": 0.11035721412891639, | |
| "grad_norm": 0.46850871504202196, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0089, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.01421830172850085, | |
| "epoch": 0.1117541408900419, | |
| "grad_norm": 0.2694884119514331, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0087, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5048.066615513393, | |
| "epoch": 0.11315106765116743, | |
| "grad_norm": 0.4818094588291497, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0003, | |
| "num_tokens": 26188824.0, | |
| "reward": 0.4587076764021601, | |
| "reward_std": 0.3670096014227186, | |
| "rewards/reward_func": 0.45870765511478695, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.014440144545265607, | |
| "epoch": 0.11454799441229295, | |
| "grad_norm": 39.00637105645605, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0041, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.014427063853612967, | |
| "epoch": 0.11594492117341848, | |
| "grad_norm": 0.9150065270465777, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0009, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.015046841731028897, | |
| "epoch": 0.117341847934544, | |
| "grad_norm": 0.24009263771304865, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0007, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 4776.74507359096, | |
| "epoch": 0.11873877469566953, | |
| "grad_norm": 0.4772391378994009, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0175, | |
| "num_tokens": 27213245.0, | |
| "reward": 0.46243097526686533, | |
| "reward_std": 0.3514525145292282, | |
| "rewards/reward_func": 0.4624309518507549, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.013508185378408857, | |
| "epoch": 0.12013570145679506, | |
| "grad_norm": 166639.4843736821, | |
| "learning_rate": 1e-06, | |
| "loss": 7.3176, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.013775874627754092, | |
| "epoch": 0.12153262821792057, | |
| "grad_norm": 27.738301939542676, | |
| "learning_rate": 1e-06, | |
| "loss": 0.019, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.014050438667514495, | |
| "epoch": 0.1229295549790461, | |
| "grad_norm": 2.3910898137109857, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0182, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 4914.035984584263, | |
| "epoch": 0.12432648174017162, | |
| "grad_norm": 0.5313282819608155, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0136, | |
| "num_tokens": 28265422.0, | |
| "reward": 0.493380460355963, | |
| "reward_std": 0.39978101530245375, | |
| "rewards/reward_func": 0.4933804316180093, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.017403810085462674, | |
| "epoch": 0.12572340850129715, | |
| "grad_norm": 2.5053609302596263, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0153, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.018204916534679278, | |
| "epoch": 0.12712033526242267, | |
| "grad_norm": 1.3673142623484746, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0147, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.019155576026865413, | |
| "epoch": 0.12851726202354818, | |
| "grad_norm": 0.285245341653243, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0145, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 4607.285940987723, | |
| "epoch": 0.12991418878467373, | |
| "grad_norm": 0.5382478014891808, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0251, | |
| "num_tokens": 29256889.0, | |
| "reward": 0.4975069910287857, | |
| "reward_std": 0.39376414673668997, | |
| "rewards/reward_func": 0.4975069612264633, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.017131159414670298, | |
| "epoch": 0.13131111554579925, | |
| "grad_norm": 277.06893594236533, | |
| "learning_rate": 1e-06, | |
| "loss": 0.041, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.017470890456544503, | |
| "epoch": 0.13270804230692476, | |
| "grad_norm": 0.595127984757126, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0264, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.017918821956430162, | |
| "epoch": 0.13410496906805028, | |
| "grad_norm": 0.3727932311400719, | |
| "learning_rate": 1e-06, | |
| "loss": 0.026, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 5105.030866350447, | |
| "epoch": 0.13550189582917582, | |
| "grad_norm": 0.5204808748332284, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0166, | |
| "num_tokens": 30346587.0, | |
| "reward": 0.40901294563497814, | |
| "reward_std": 0.3415069899388722, | |
| "rewards/reward_func": 0.4090129222188677, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0174449899766062, | |
| "epoch": 0.13689882259030134, | |
| "grad_norm": 1.1528181627310667, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0183, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.01785454393497535, | |
| "epoch": 0.13829574935142686, | |
| "grad_norm": 0.6193098655808096, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0177, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.018185677073363746, | |
| "epoch": 0.13969267611255237, | |
| "grad_norm": 0.29035255969562146, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0174, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3904.9848981584823, | |
| "epoch": 0.14108960287367792, | |
| "grad_norm": 0.709202941284989, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0147, | |
| "num_tokens": 31202372.0, | |
| "reward": 0.529426036136491, | |
| "reward_std": 0.35220117547682356, | |
| "rewards/reward_func": 0.5294260127203805, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.01801375742070377, | |
| "epoch": 0.14248652963480343, | |
| "grad_norm": 1.0665575599442703, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0166, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.017245101110477532, | |
| "epoch": 0.14388345639592895, | |
| "grad_norm": 0.5821159923874696, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0158, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.01721734370637153, | |
| "epoch": 0.1452803831570545, | |
| "grad_norm": 0.4938223790926132, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0153, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 4391.734933035715, | |
| "epoch": 0.14667730991818, | |
| "grad_norm": 0.6621058895011234, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0067, | |
| "num_tokens": 32151129.0, | |
| "reward": 0.5155340305396489, | |
| "reward_std": 0.36261362050260815, | |
| "rewards/reward_func": 0.5155340198959623, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.021650432209883417, | |
| "epoch": 0.14807423667930553, | |
| "grad_norm": 7.221717083371915, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0095, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_ratio": 0.022131933498063257, | |
| "epoch": 0.14947116344043104, | |
| "grad_norm": 3.9217845571015726, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0087, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_ratio": 0.023090457543730736, | |
| "epoch": 0.1508680902015566, | |
| "grad_norm": 1.0915253624674428, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0083, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 4265.617579868862, | |
| "epoch": 0.1522650169626821, | |
| "grad_norm": 0.7674370802581663, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0076, | |
| "num_tokens": 33074935.0, | |
| "reward": 0.49383693294865744, | |
| "reward_std": 0.3673490881919861, | |
| "rewards/reward_func": 0.49383691059691565, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_ratio": 0.01965414373470204, | |
| "epoch": 0.15366194372380762, | |
| "grad_norm": 3.114603095736279, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0099, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.020012260481183018, | |
| "epoch": 0.15505887048493314, | |
| "grad_norm": 4.049634875724923, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0094, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_ratio": 0.02028396732306906, | |
| "epoch": 0.15645579724605868, | |
| "grad_norm": 0.5429450253769021, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0089, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3556.7655116489955, | |
| "epoch": 0.1578527240071842, | |
| "grad_norm": 0.8213126340122913, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0142, | |
| "num_tokens": 33860148.0, | |
| "reward": 0.520222473357405, | |
| "reward_std": 0.3601721642272813, | |
| "rewards/reward_func": 0.5202224499412945, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_ratio": 0.021029458713850806, | |
| "epoch": 0.15924965076830971, | |
| "grad_norm": 511.99698568152866, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0391, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_ratio": 0.020824986709547893, | |
| "epoch": 0.16064657752943523, | |
| "grad_norm": 1.6554233851238163, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0163, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.021296620901141847, | |
| "epoch": 0.16204350429056077, | |
| "grad_norm": 5.389449179546646, | |
| "learning_rate": 1e-06, | |
| "loss": 0.016, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 4172.214425223215, | |
| "epoch": 0.1634404310516863, | |
| "grad_norm": 0.7951777611207881, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0188, | |
| "num_tokens": 34766754.0, | |
| "reward": 0.43932128697633743, | |
| "reward_std": 0.36100248460258755, | |
| "rewards/reward_func": 0.4393212656889643, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_ratio": 0.024627858506781713, | |
| "epoch": 0.1648373578128118, | |
| "grad_norm": 5.497957714291199, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0267, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_ratio": 0.024561051279306412, | |
| "epoch": 0.16623428457393732, | |
| "grad_norm": 4.006479624309579, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0259, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_ratio": 0.02518816019541451, | |
| "epoch": 0.16763121133506287, | |
| "grad_norm": 5.696382550533168, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0217, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3450.438938685826, | |
| "epoch": 0.16902813809618839, | |
| "grad_norm": 0.8756816112591876, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0128, | |
| "num_tokens": 35530451.0, | |
| "reward": 0.4647199107067926, | |
| "reward_std": 0.35261689871549606, | |
| "rewards/reward_func": 0.46471990112747463, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_ratio": 0.023490584788045714, | |
| "epoch": 0.1704250648573139, | |
| "grad_norm": 8.184182062225446, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0176, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_ratio": 0.022877583213682686, | |
| "epoch": 0.17182199161843945, | |
| "grad_norm": 2.337796998952532, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0158, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_ratio": 0.022725746002314345, | |
| "epoch": 0.17321891837956496, | |
| "grad_norm": 1.204853002031202, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0149, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3755.143040248326, | |
| "epoch": 0.17461584514069048, | |
| "grad_norm": 1.1667266762961181, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0009, | |
| "num_tokens": 36355620.0, | |
| "reward": 0.48860155258859905, | |
| "reward_std": 0.3995990295495306, | |
| "rewards/reward_func": 0.4886015323655946, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.024759572331926653, | |
| "epoch": 0.176012771901816, | |
| "grad_norm": 11.549731072093149, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0075, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_ratio": 0.023941728619060347, | |
| "epoch": 0.17740969866294154, | |
| "grad_norm": 1.5478778519158778, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0043, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_ratio": 0.023272603883274963, | |
| "epoch": 0.17880662542406706, | |
| "grad_norm": 1.0384027782585576, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0034, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3561.3930053710938, | |
| "epoch": 0.18020355218519257, | |
| "grad_norm": 1.203850582780552, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0059, | |
| "num_tokens": 37143444.0, | |
| "reward": 0.4855222978762218, | |
| "reward_std": 0.38453598746231626, | |
| "rewards/reward_func": 0.4855222872325352, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_ratio": 0.024004271253943443, | |
| "epoch": 0.1816004789463181, | |
| "grad_norm": 11.686104385649747, | |
| "learning_rate": 1e-06, | |
| "loss": 0.012, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.02320788980328611, | |
| "epoch": 0.18299740570744363, | |
| "grad_norm": 51.97949282308589, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0129, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_ratio": 0.023830042220652103, | |
| "epoch": 0.18439433246856915, | |
| "grad_norm": 77704.4679582965, | |
| "learning_rate": 1e-06, | |
| "loss": 0.929, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3375.382795061384, | |
| "epoch": 0.18579125922969467, | |
| "grad_norm": 1.1905136253909123, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0195, | |
| "num_tokens": 37893570.0, | |
| "reward": 0.4746183859450476, | |
| "reward_std": 0.41642780708415167, | |
| "rewards/reward_func": 0.47461836465767454, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_ratio": 0.03054676218224423, | |
| "epoch": 0.18718818599082018, | |
| "grad_norm": 47.607522363933455, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0306, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_ratio": 0.03216503720198359, | |
| "epoch": 0.18858511275194573, | |
| "grad_norm": 6.197384439543361, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0265, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.03178960005087512, | |
| "epoch": 0.18998203951307124, | |
| "grad_norm": 2.236284664240462, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0236, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3824.760410853795, | |
| "epoch": 0.19137896627419676, | |
| "grad_norm": 1.0181749371365818, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0301, | |
| "num_tokens": 38731414.0, | |
| "reward": 0.4555432179144451, | |
| "reward_std": 0.371168036546026, | |
| "rewards/reward_func": 0.45554319130522863, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_ratio": 0.02426370605826378, | |
| "epoch": 0.1927758930353223, | |
| "grad_norm": 15.973977289606324, | |
| "learning_rate": 1e-06, | |
| "loss": 2.1765, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_ratio": 0.023818871338984797, | |
| "epoch": 0.19417281979644782, | |
| "grad_norm": 125.42621828076582, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0544, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_ratio": 0.0235800734455032, | |
| "epoch": 0.19556974655757334, | |
| "grad_norm": 5.708397430999111, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0358, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3121.5970720563614, | |
| "epoch": 0.19696667331869885, | |
| "grad_norm": 0.9463162061122932, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0074, | |
| "num_tokens": 39432472.0, | |
| "reward": 0.3878828393561499, | |
| "reward_std": 0.32353205340249197, | |
| "rewards/reward_func": 0.3878828233906201, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_ratio": 0.031209798529744148, | |
| "epoch": 0.1983636000798244, | |
| "grad_norm": 19.178703569706425, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0158, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_ratio": 0.031645855334188254, | |
| "epoch": 0.19976052684094991, | |
| "grad_norm": 73.92547009994209, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0183, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_ratio": 0.032449710848076005, | |
| "epoch": 0.20115745360207543, | |
| "grad_norm": 5.770213423174226, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0164, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3645.8165196010045, | |
| "epoch": 0.20255438036320095, | |
| "grad_norm": 1.2228735898177363, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0139, | |
| "num_tokens": 40234738.0, | |
| "reward": 0.4959501911486898, | |
| "reward_std": 0.3776449923004423, | |
| "rewards/reward_func": 0.4959501826337406, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.029987079756600515, | |
| "epoch": 0.2039513071243265, | |
| "grad_norm": 45.612373429574255, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0274, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_ratio": 0.029181892717523233, | |
| "epoch": 0.205348233885452, | |
| "grad_norm": 61.829029657734615, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0238, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_ratio": 0.028760566908333982, | |
| "epoch": 0.20674516064657752, | |
| "grad_norm": 6.854234290541286, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0212, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3616.0256783621653, | |
| "epoch": 0.20814208740770304, | |
| "grad_norm": 1.1080044572109065, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0188, | |
| "num_tokens": 41031965.0, | |
| "reward": 0.37510563326733454, | |
| "reward_std": 0.35230499718870434, | |
| "rewards/reward_func": 0.37510561889835764, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_ratio": 0.032221541074769836, | |
| "epoch": 0.20953901416882859, | |
| "grad_norm": 22328.844087735877, | |
| "learning_rate": 1e-06, | |
| "loss": 0.7198, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.03243154074464526, | |
| "epoch": 0.2109359409299541, | |
| "grad_norm": 11006.630246836325, | |
| "learning_rate": 1e-06, | |
| "loss": 1.3521, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_ratio": 0.032671846981559484, | |
| "epoch": 0.21233286769107962, | |
| "grad_norm": 52.44059977916511, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0309, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3217.7909197126114, | |
| "epoch": 0.21372979445220514, | |
| "grad_norm": 1.3203188901357052, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0184, | |
| "num_tokens": 41750731.0, | |
| "reward": 0.4333155059388706, | |
| "reward_std": 0.3546490435089384, | |
| "rewards/reward_func": 0.43331548997334074, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_ratio": 0.032571413687297275, | |
| "epoch": 0.21512672121333068, | |
| "grad_norm": 56932.32164959619, | |
| "learning_rate": 1e-06, | |
| "loss": 1.4737, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_ratio": 0.03264636732637882, | |
| "epoch": 0.2165236479744562, | |
| "grad_norm": 569.4118154787187, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0464, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.033495722870741575, | |
| "epoch": 0.2179205747355817, | |
| "grad_norm": 14.372919673445498, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0297, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2904.224591936384, | |
| "epoch": 0.21931750149670726, | |
| "grad_norm": 1.4884091255614038, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0027, | |
| "num_tokens": 42407374.0, | |
| "reward": 0.46237824857234955, | |
| "reward_std": 0.38831384958965437, | |
| "rewards/reward_func": 0.4623782304780824, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_ratio": 0.03049316949078015, | |
| "epoch": 0.22071442825783277, | |
| "grad_norm": 753.2718719394906, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0615, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_ratio": 0.028818647909377302, | |
| "epoch": 0.2221113550189583, | |
| "grad_norm": 193.7516952886048, | |
| "learning_rate": 1e-06, | |
| "loss": 0.03, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_ratio": 0.028797397390007973, | |
| "epoch": 0.2235082817800838, | |
| "grad_norm": 86.17864803711156, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0187, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2706.137860979353, | |
| "epoch": 0.22490520854120935, | |
| "grad_norm": 1.437051783056985, | |
| "learning_rate": 1e-06, | |
| "loss": 0.004, | |
| "num_tokens": 43026529.0, | |
| "reward": 0.40501056824411663, | |
| "reward_std": 0.3891604415008, | |
| "rewards/reward_func": 0.4050105522785868, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_ratio": 0.03492548343326364, | |
| "epoch": 0.22630213530233487, | |
| "grad_norm": 1637.6323089099733, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0796, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_ratio": 0.034652630399380414, | |
| "epoch": 0.22769906206346038, | |
| "grad_norm": 294.8538016270806, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0397, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_ratio": 0.03514658712915012, | |
| "epoch": 0.2290959888245859, | |
| "grad_norm": 1729.0881090208584, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1081, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3302.3215855189733, | |
| "epoch": 0.23049291558571144, | |
| "grad_norm": 1.3478961700451182, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0095, | |
| "num_tokens": 43761689.0, | |
| "reward": 0.354110734803336, | |
| "reward_std": 0.3442910239100456, | |
| "rewards/reward_func": 0.35411071511251585, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.03175032351698194, | |
| "epoch": 0.23188984234683696, | |
| "grad_norm": 34.2352104129209, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0236, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_ratio": 0.031168780555682524, | |
| "epoch": 0.23328676910796248, | |
| "grad_norm": 3479.355422028937, | |
| "learning_rate": 1e-06, | |
| "loss": 0.3311, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_ratio": 0.031411009441529004, | |
| "epoch": 0.234683695869088, | |
| "grad_norm": 211.96486688673437, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0428, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3288.581760951451, | |
| "epoch": 0.23608062263021354, | |
| "grad_norm": 1.5809513920036224, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0206, | |
| "num_tokens": 44495433.0, | |
| "reward": 0.3336522286491735, | |
| "reward_std": 0.35370165216071264, | |
| "rewards/reward_func": 0.3336522126836436, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_ratio": 0.03352949349209666, | |
| "epoch": 0.23747754939133905, | |
| "grad_norm": 131.25750741813656, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0411, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.033907221497169564, | |
| "epoch": 0.23887447615246457, | |
| "grad_norm": 38.44427070292949, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0333, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_ratio": 0.034381040039339235, | |
| "epoch": 0.24027140291359012, | |
| "grad_norm": 17107451499810.188, | |
| "learning_rate": 1e-06, | |
| "loss": 549430272.0, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3361.0919799804688, | |
| "epoch": 0.24166832967471563, | |
| "grad_norm": 1.637145998911673, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0187, | |
| "num_tokens": 45243898.0, | |
| "reward": 0.394156134554318, | |
| "reward_std": 0.40689716062375475, | |
| "rewards/reward_func": 0.3941561228462628, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_ratio": 0.032513607027275224, | |
| "epoch": 0.24306525643584115, | |
| "grad_norm": 800.8287273060004, | |
| "learning_rate": 1e-06, | |
| "loss": 1.6832, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_ratio": 0.030495245408798968, | |
| "epoch": 0.24446218319696666, | |
| "grad_norm": 729.365666028725, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1281, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.03067155448453767, | |
| "epoch": 0.2458591099580922, | |
| "grad_norm": 257.8929935202217, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0783, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2669.938886369978, | |
| "epoch": 0.24725603671921773, | |
| "grad_norm": 1.6984457904054797, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0017, | |
| "num_tokens": 45856081.0, | |
| "reward": 0.3272132841604097, | |
| "reward_std": 0.36115063620465143, | |
| "rewards/reward_func": 0.3272132767098291, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_ratio": 0.03918117896786758, | |
| "epoch": 0.24865296348034324, | |
| "grad_norm": 459.0007426096092, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0378, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_ratio": 0.037612543175263066, | |
| "epoch": 0.2500498902414688, | |
| "grad_norm": 198.16000110545232, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0357, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_ratio": 0.03760030067392758, | |
| "epoch": 0.2514468170025943, | |
| "grad_norm": 193.31294839005466, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0318, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3767.30623953683, | |
| "epoch": 0.2528437437637198, | |
| "grad_norm": 1.4505504547674242, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0004, | |
| "num_tokens": 46683920.0, | |
| "reward": 0.3422575050166675, | |
| "reward_std": 0.3818031592028482, | |
| "rewards/reward_func": 0.3422574890511377, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_ratio": 0.03315202798694372, | |
| "epoch": 0.25424067052484534, | |
| "grad_norm": 238.64962645549264, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0462, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_ratio": 0.03287409871284451, | |
| "epoch": 0.25563759728597085, | |
| "grad_norm": 6799.648792237924, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0493, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_ratio": 0.03349324116217239, | |
| "epoch": 0.25703452404709637, | |
| "grad_norm": 1395.3440659332787, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1532, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3256.025630405971, | |
| "epoch": 0.2584314508082219, | |
| "grad_norm": 1.8519382626688905, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0091, | |
| "num_tokens": 47409833.0, | |
| "reward": 0.32464135118893217, | |
| "reward_std": 0.3450521251985005, | |
| "rewards/reward_func": 0.324641336287771, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.034153553524187634, | |
| "epoch": 0.25982837756934746, | |
| "grad_norm": 55.33553677644012, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0322, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_ratio": 0.03325902218265193, | |
| "epoch": 0.261225304330473, | |
| "grad_norm": 807.7840491437855, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0771, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_ratio": 0.032821542982544215, | |
| "epoch": 0.2626222310915985, | |
| "grad_norm": 1395.5436294619333, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0888, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3214.1480887276784, | |
| "epoch": 0.264019157852724, | |
| "grad_norm": 1.7683494272507614, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0112, | |
| "num_tokens": 48129036.0, | |
| "reward": 0.31755988619157244, | |
| "reward_std": 0.3383456287639482, | |
| "rewards/reward_func": 0.31755986117890905, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_ratio": 0.03975264209189585, | |
| "epoch": 0.2654160846138495, | |
| "grad_norm": 45184.22194084358, | |
| "learning_rate": 1e-06, | |
| "loss": 0.4443, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.03915290268404143, | |
| "epoch": 0.26681301137497504, | |
| "grad_norm": 1242.8816256939144, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1447, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_ratio": 0.03953878129167216, | |
| "epoch": 0.26820993813610056, | |
| "grad_norm": 1479.8321627428613, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1168, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 3272.2399553571427, | |
| "epoch": 0.26960686489722613, | |
| "grad_norm": 2.177506823211759, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0092, | |
| "num_tokens": 48860342.0, | |
| "reward": 0.34522383979388643, | |
| "reward_std": 0.3796129695006779, | |
| "rewards/reward_func": 0.34522382276398794, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_ratio": 0.03925856229450021, | |
| "epoch": 0.27100379165835164, | |
| "grad_norm": 513.8290574797221, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0613, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_ratio": 0.03799283770578248, | |
| "epoch": 0.27240071841947716, | |
| "grad_norm": 8983.081819712857, | |
| "learning_rate": 1e-06, | |
| "loss": 0.3161, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.036926204205623696, | |
| "epoch": 0.2737976451806027, | |
| "grad_norm": 1027.0780848048523, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1274, | |
| "step": 196 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 4028.9797712053573, | |
| "epoch": 0.2751945719417282, | |
| "grad_norm": 1.3698484721094724, | |
| "learning_rate": 1e-06, | |
| "loss": 0.018, | |
| "num_tokens": 49737682.0, | |
| "reward": 0.2579219543508121, | |
| "reward_std": 0.32817436435392927, | |
| "rewards/reward_func": 0.25792194423930986, | |
| "step": 197 | |
| }, | |
| { | |
| "clip_ratio": 0.03714690676757267, | |
| "epoch": 0.2765914987028537, | |
| "grad_norm": 759.0060595947047, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0657, | |
| "step": 198 | |
| }, | |
| { | |
| "clip_ratio": 0.0380573890038899, | |
| "epoch": 0.2779884254639792, | |
| "grad_norm": 7489.486866588998, | |
| "learning_rate": 1e-06, | |
| "loss": 0.9186, | |
| "step": 199 | |
| }, | |
| { | |
| "clip_ratio": 0.03898970350357039, | |
| "epoch": 0.27938535222510474, | |
| "grad_norm": 403.4437322167033, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1357, | |
| "step": 200 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 715, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |