diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,9410 @@ +{ + "best_global_step": 300, + "best_metric": 0.42507252, + "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_dpo_round2_0dot9_0603/v1-20250603-223558/checkpoint-300", + "epoch": 2.985945945945946, + "eval_steps": 300, + "global_step": 519, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.005765765765765766, + "grad_norm": 2.1474921703338623, + "learning_rate": 7.692307692307694e-06, + "logits/chosen": -0.4285973310470581, + "logits/rejected": -0.721319854259491, + "logps/chosen": -2.468954086303711, + "logps/rejected": -59.87730407714844, + "loss": 0.22946561872959137, + "memory(GiB)": 30.65, + "nll_loss": 0.13756254315376282, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.23724405467510223, + "rewards/margins": 3.9806408882141113, + "rewards/rejected": -3.743396759033203, + "step": 1, + "train_speed(iter/s)": 0.01022 + }, + { + "epoch": 0.011531531531531532, + "grad_norm": 2.328110456466675, + "learning_rate": 1.5384615384615387e-05, + "logits/chosen": -0.3982561528682709, + "logits/rejected": -0.7214521169662476, + "logps/chosen": -3.335645914077759, + "logps/rejected": -60.79209899902344, + "loss": 0.3770241439342499, + "memory(GiB)": 30.65, + "nll_loss": 0.23075783252716064, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10687967389822006, + "rewards/margins": 3.9392333030700684, + "rewards/rejected": -3.832353353500366, + "step": 2, + "train_speed(iter/s)": 0.010488 + }, + { + "epoch": 0.017297297297297298, + "grad_norm": 14.568985939025879, + "learning_rate": 2.307692307692308e-05, + "logits/chosen": -0.43175598978996277, + "logits/rejected": -0.6921666264533997, + "logps/chosen": -5.190546035766602, + "logps/rejected": -51.33646011352539, + "loss": 0.9886025786399841, + "memory(GiB)": 34.1, + "nll_loss": 0.6371194124221802, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.034887176007032394, + "rewards/margins": 3.2021734714508057, + "rewards/rejected": -3.237060785293579, + "step": 3, + "train_speed(iter/s)": 0.010581 + }, + { + "epoch": 0.023063063063063063, + "grad_norm": 13.817249298095703, + "learning_rate": 3.0769230769230774e-05, + "logits/chosen": -0.4616771340370178, + "logits/rejected": -0.6264235973358154, + "logps/chosen": -3.2552552223205566, + "logps/rejected": -51.25586700439453, + "loss": 0.5021163821220398, + "memory(GiB)": 37.58, + "nll_loss": 0.26375898718833923, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.06816040724515915, + "rewards/margins": 3.5297036170959473, + "rewards/rejected": -3.461543083190918, + "step": 4, + "train_speed(iter/s)": 0.010625 + }, + { + "epoch": 0.02882882882882883, + "grad_norm": 2.363553524017334, + "learning_rate": 3.846153846153846e-05, + "logits/chosen": -0.48805952072143555, + "logits/rejected": -0.7040677070617676, + "logps/chosen": -6.89398717880249, + "logps/rejected": -59.37879943847656, + "loss": 0.39657193422317505, + "memory(GiB)": 37.58, + "nll_loss": 0.2818652391433716, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1999940723180771, + "rewards/margins": 4.230030059814453, + "rewards/rejected": -4.030035495758057, + "step": 5, + "train_speed(iter/s)": 0.010659 + }, + { + "epoch": 0.034594594594594595, + "grad_norm": 4.551424026489258, + "learning_rate": 4.615384615384616e-05, + "logits/chosen": -0.39471542835235596, + "logits/rejected": -0.6508359909057617, + "logps/chosen": -6.633285999298096, + "logps/rejected": -58.268653869628906, + "loss": 0.5636770129203796, + "memory(GiB)": 41.24, + "nll_loss": 0.41472452878952026, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1396198272705078, + "rewards/margins": 3.7723348140716553, + "rewards/rejected": -3.6327152252197266, + "step": 6, + "train_speed(iter/s)": 0.010667 + }, + { + "epoch": 0.04036036036036036, + "grad_norm": 4.302145481109619, + "learning_rate": 5.384615384615385e-05, + "logits/chosen": -0.42404329776763916, + "logits/rejected": -0.7084805369377136, + "logps/chosen": -4.09792947769165, + "logps/rejected": -69.69088745117188, + "loss": 0.5202179551124573, + "memory(GiB)": 41.24, + "nll_loss": 0.3875095546245575, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.018997177481651306, + "rewards/margins": 5.0437703132629395, + "rewards/rejected": -5.062767505645752, + "step": 7, + "train_speed(iter/s)": 0.010685 + }, + { + "epoch": 0.04612612612612613, + "grad_norm": 4.475785255432129, + "learning_rate": 6.153846153846155e-05, + "logits/chosen": -0.4411894977092743, + "logits/rejected": -0.6594015955924988, + "logps/chosen": -6.067828178405762, + "logps/rejected": -52.161441802978516, + "loss": 0.8079419136047363, + "memory(GiB)": 41.24, + "nll_loss": 0.5527109503746033, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.13843362033367157, + "rewards/margins": 3.638244390487671, + "rewards/rejected": -3.4998104572296143, + "step": 8, + "train_speed(iter/s)": 0.010699 + }, + { + "epoch": 0.05189189189189189, + "grad_norm": 7.412286281585693, + "learning_rate": 6.923076923076924e-05, + "logits/chosen": -0.4308490753173828, + "logits/rejected": -0.6868806481361389, + "logps/chosen": -4.5601372718811035, + "logps/rejected": -75.88018798828125, + "loss": 0.3704576790332794, + "memory(GiB)": 41.24, + "nll_loss": 0.23506566882133484, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1938612163066864, + "rewards/margins": 5.914104461669922, + "rewards/rejected": -5.720243453979492, + "step": 9, + "train_speed(iter/s)": 0.010708 + }, + { + "epoch": 0.05765765765765766, + "grad_norm": 2.0841763019561768, + "learning_rate": 7.692307692307693e-05, + "logits/chosen": -0.4546598792076111, + "logits/rejected": -0.7517622113227844, + "logps/chosen": -1.471292495727539, + "logps/rejected": -70.88200378417969, + "loss": 0.2737431526184082, + "memory(GiB)": 41.24, + "nll_loss": 0.14067727327346802, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13278257846832275, + "rewards/margins": 5.628582000732422, + "rewards/rejected": -5.495799541473389, + "step": 10, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.06342342342342343, + "grad_norm": 5.912591457366943, + "learning_rate": 8.461538461538461e-05, + "logits/chosen": -0.4128813147544861, + "logits/rejected": -0.5927000045776367, + "logps/chosen": -7.319942474365234, + "logps/rejected": -61.27806091308594, + "loss": 0.5680095553398132, + "memory(GiB)": 41.24, + "nll_loss": 0.3753523826599121, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07616602629423141, + "rewards/margins": 4.830938816070557, + "rewards/rejected": -4.754772663116455, + "step": 11, + "train_speed(iter/s)": 0.010727 + }, + { + "epoch": 0.06918918918918919, + "grad_norm": 2.5374209880828857, + "learning_rate": 9.230769230769232e-05, + "logits/chosen": -0.349031925201416, + "logits/rejected": -0.6373917460441589, + "logps/chosen": -4.199922561645508, + "logps/rejected": -72.83607482910156, + "loss": 0.34027042984962463, + "memory(GiB)": 41.24, + "nll_loss": 0.22808796167373657, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.20608201622962952, + "rewards/margins": 5.424815654754639, + "rewards/rejected": -5.218733787536621, + "step": 12, + "train_speed(iter/s)": 0.010728 + }, + { + "epoch": 0.07495495495495495, + "grad_norm": 2.7130517959594727, + "learning_rate": 0.0001, + "logits/chosen": -0.3631379306316376, + "logits/rejected": -0.6601940989494324, + "logps/chosen": -3.3938565254211426, + "logps/rejected": -65.2313232421875, + "loss": 0.4514213800430298, + "memory(GiB)": 41.24, + "nll_loss": 0.30333614349365234, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06806281208992004, + "rewards/margins": 4.726724624633789, + "rewards/rejected": -4.658661365509033, + "step": 13, + "train_speed(iter/s)": 0.010733 + }, + { + "epoch": 0.08072072072072072, + "grad_norm": 1.5474798679351807, + "learning_rate": 0.0001076923076923077, + "logits/chosen": -0.4096493124961853, + "logits/rejected": -0.6515597105026245, + "logps/chosen": -2.300206184387207, + "logps/rejected": -57.26811218261719, + "loss": 0.3060320019721985, + "memory(GiB)": 41.24, + "nll_loss": 0.1727571338415146, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08898697793483734, + "rewards/margins": 4.1406354904174805, + "rewards/rejected": -4.0516486167907715, + "step": 14, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 0.08648648648648649, + "grad_norm": 1.628075361251831, + "learning_rate": 0.00011538461538461538, + "logits/chosen": -0.4402551054954529, + "logits/rejected": -0.6672070622444153, + "logps/chosen": -2.9995224475860596, + "logps/rejected": -53.87057113647461, + "loss": 0.3575773239135742, + "memory(GiB)": 41.24, + "nll_loss": 0.2217392474412918, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.20550209283828735, + "rewards/margins": 3.6874935626983643, + "rewards/rejected": -3.4819912910461426, + "step": 15, + "train_speed(iter/s)": 0.010742 + }, + { + "epoch": 0.09225225225225225, + "grad_norm": 1.838375449180603, + "learning_rate": 0.0001230769230769231, + "logits/chosen": -0.3547935485839844, + "logits/rejected": -0.6626885533332825, + "logps/chosen": -4.558413982391357, + "logps/rejected": -64.36922454833984, + "loss": 0.45015937089920044, + "memory(GiB)": 41.24, + "nll_loss": 0.3128659129142761, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14241908490657806, + "rewards/margins": 4.83186149597168, + "rewards/rejected": -4.689442157745361, + "step": 16, + "train_speed(iter/s)": 0.010745 + }, + { + "epoch": 0.09801801801801802, + "grad_norm": 3.119417667388916, + "learning_rate": 0.00013076923076923077, + "logits/chosen": -0.3334687352180481, + "logits/rejected": -0.6293997168540955, + "logps/chosen": -5.949204444885254, + "logps/rejected": -74.25395965576172, + "loss": 0.6850419044494629, + "memory(GiB)": 41.24, + "nll_loss": 0.5004001259803772, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07516612857580185, + "rewards/margins": 4.788809776306152, + "rewards/rejected": -4.713644027709961, + "step": 17, + "train_speed(iter/s)": 0.010741 + }, + { + "epoch": 0.10378378378378378, + "grad_norm": 1.6356091499328613, + "learning_rate": 0.00013846153846153847, + "logits/chosen": -0.3526935875415802, + "logits/rejected": -0.6357190608978271, + "logps/chosen": -2.911073923110962, + "logps/rejected": -66.46131896972656, + "loss": 0.32070592045783997, + "memory(GiB)": 41.24, + "nll_loss": 0.18397927284240723, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09694083034992218, + "rewards/margins": 5.153117656707764, + "rewards/rejected": -5.056177139282227, + "step": 18, + "train_speed(iter/s)": 0.010741 + }, + { + "epoch": 0.10954954954954955, + "grad_norm": 2.7403504848480225, + "learning_rate": 0.00014615384615384615, + "logits/chosen": -0.3763662278652191, + "logits/rejected": -0.586420476436615, + "logps/chosen": -4.417596340179443, + "logps/rejected": -58.99639892578125, + "loss": 0.4634058177471161, + "memory(GiB)": 41.24, + "nll_loss": 0.28715017437934875, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1435782015323639, + "rewards/margins": 4.609218120574951, + "rewards/rejected": -4.465640068054199, + "step": 19, + "train_speed(iter/s)": 0.010743 + }, + { + "epoch": 0.11531531531531532, + "grad_norm": 1.982851266860962, + "learning_rate": 0.00015384615384615385, + "logits/chosen": -0.3701193332672119, + "logits/rejected": -0.6088087558746338, + "logps/chosen": -4.619842529296875, + "logps/rejected": -68.1020278930664, + "loss": 0.4344192445278168, + "memory(GiB)": 41.24, + "nll_loss": 0.26591166853904724, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10610304027795792, + "rewards/margins": 5.044137954711914, + "rewards/rejected": -4.938034534454346, + "step": 20, + "train_speed(iter/s)": 0.010745 + }, + { + "epoch": 0.12108108108108108, + "grad_norm": 3.175961494445801, + "learning_rate": 0.00016153846153846155, + "logits/chosen": -0.40009674429893494, + "logits/rejected": -0.642030656337738, + "logps/chosen": -3.357761859893799, + "logps/rejected": -65.21231842041016, + "loss": 0.40142354369163513, + "memory(GiB)": 41.24, + "nll_loss": 0.2889588177204132, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1323939859867096, + "rewards/margins": 5.234187602996826, + "rewards/rejected": -5.1017937660217285, + "step": 21, + "train_speed(iter/s)": 0.010748 + }, + { + "epoch": 0.12684684684684686, + "grad_norm": 1.9400721788406372, + "learning_rate": 0.00016923076923076923, + "logits/chosen": -0.3025435209274292, + "logits/rejected": -0.6950021386146545, + "logps/chosen": -2.492130994796753, + "logps/rejected": -107.03312683105469, + "loss": 0.2920421361923218, + "memory(GiB)": 41.24, + "nll_loss": 0.19772115349769592, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.19073158502578735, + "rewards/margins": 8.599780082702637, + "rewards/rejected": -8.409048080444336, + "step": 22, + "train_speed(iter/s)": 0.010746 + }, + { + "epoch": 0.13261261261261262, + "grad_norm": 1.3099267482757568, + "learning_rate": 0.00017692307692307693, + "logits/chosen": -0.36498135328292847, + "logits/rejected": -0.6359634399414062, + "logps/chosen": -2.421377658843994, + "logps/rejected": -74.05368041992188, + "loss": 0.20869436860084534, + "memory(GiB)": 41.24, + "nll_loss": 0.11023169010877609, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18500684201717377, + "rewards/margins": 5.958514213562012, + "rewards/rejected": -5.773507118225098, + "step": 23, + "train_speed(iter/s)": 0.010747 + }, + { + "epoch": 0.13837837837837838, + "grad_norm": 1.9334064722061157, + "learning_rate": 0.00018461538461538463, + "logits/chosen": -0.45037421584129333, + "logits/rejected": -0.7719848155975342, + "logps/chosen": -2.6665430068969727, + "logps/rejected": -94.5350570678711, + "loss": 0.3057625889778137, + "memory(GiB)": 41.24, + "nll_loss": 0.2374470978975296, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09573014080524445, + "rewards/margins": 8.040020942687988, + "rewards/rejected": -7.944291114807129, + "step": 24, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.14414414414414414, + "grad_norm": 7.789314270019531, + "learning_rate": 0.00019230769230769233, + "logits/chosen": -0.3473276197910309, + "logits/rejected": -0.6793957352638245, + "logps/chosen": -4.235269069671631, + "logps/rejected": -109.01844024658203, + "loss": 0.42337244749069214, + "memory(GiB)": 41.24, + "nll_loss": 0.21255403757095337, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.009569963440299034, + "rewards/margins": 8.769816398620605, + "rewards/rejected": -8.760246276855469, + "step": 25, + "train_speed(iter/s)": 0.010747 + }, + { + "epoch": 0.1499099099099099, + "grad_norm": 1.970948338508606, + "learning_rate": 0.0002, + "logits/chosen": -0.40258127450942993, + "logits/rejected": -0.7479668259620667, + "logps/chosen": -2.0013341903686523, + "logps/rejected": -85.51154327392578, + "loss": 0.4197818636894226, + "memory(GiB)": 41.24, + "nll_loss": 0.2574591636657715, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16396881639957428, + "rewards/margins": 7.033238410949707, + "rewards/rejected": -6.869269847869873, + "step": 26, + "train_speed(iter/s)": 0.010746 + }, + { + "epoch": 0.15567567567567567, + "grad_norm": 11.3534574508667, + "learning_rate": 0.00019999796963348897, + "logits/chosen": -0.5246711373329163, + "logits/rejected": -0.7294141054153442, + "logps/chosen": -11.85606575012207, + "logps/rejected": -76.03211975097656, + "loss": 0.8875234127044678, + "memory(GiB)": 41.24, + "nll_loss": 0.575400710105896, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.2269383817911148, + "rewards/margins": 5.498793125152588, + "rewards/rejected": -5.725731372833252, + "step": 27, + "train_speed(iter/s)": 0.010747 + }, + { + "epoch": 0.16144144144144143, + "grad_norm": 2.0314371585845947, + "learning_rate": 0.00019999187861640362, + "logits/chosen": -0.5284671187400818, + "logits/rejected": -0.7846834063529968, + "logps/chosen": -4.434022426605225, + "logps/rejected": -87.89623260498047, + "loss": 0.3868662416934967, + "memory(GiB)": 41.24, + "nll_loss": 0.30320096015930176, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07812386751174927, + "rewards/margins": 7.374299049377441, + "rewards/rejected": -7.296175479888916, + "step": 28, + "train_speed(iter/s)": 0.010748 + }, + { + "epoch": 0.16720720720720722, + "grad_norm": 6.089433193206787, + "learning_rate": 0.0001999817271960839, + "logits/chosen": -0.4792092442512512, + "logits/rejected": -0.7371792793273926, + "logps/chosen": -3.6791412830352783, + "logps/rejected": -80.48262023925781, + "loss": 0.34335166215896606, + "memory(GiB)": 41.24, + "nll_loss": 0.22308313846588135, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.049518827348947525, + "rewards/margins": 6.395692825317383, + "rewards/rejected": -6.445211410522461, + "step": 29, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.17297297297297298, + "grad_norm": 1.8896535634994507, + "learning_rate": 0.00019996751578475186, + "logits/chosen": -0.44468259811401367, + "logits/rejected": -0.7023504972457886, + "logps/chosen": -3.048004388809204, + "logps/rejected": -80.6348648071289, + "loss": 0.30949145555496216, + "memory(GiB)": 41.24, + "nll_loss": 0.1800393909215927, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15738406777381897, + "rewards/margins": 5.8682780265808105, + "rewards/rejected": -5.710893630981445, + "step": 30, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.17873873873873874, + "grad_norm": 2.7516067028045654, + "learning_rate": 0.00019994924495949504, + "logits/chosen": -0.4754641652107239, + "logits/rejected": -0.7408227920532227, + "logps/chosen": -5.943354606628418, + "logps/rejected": -79.13115692138672, + "loss": 0.6435320973396301, + "memory(GiB)": 41.24, + "nll_loss": 0.4586246609687805, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0629303902387619, + "rewards/margins": 6.110820293426514, + "rewards/rejected": -6.0478901863098145, + "step": 31, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.1845045045045045, + "grad_norm": 5.854465007781982, + "learning_rate": 0.00019992691546224278, + "logits/chosen": -0.45127877593040466, + "logits/rejected": -0.7568551898002625, + "logps/chosen": -2.736678123474121, + "logps/rejected": -66.9885025024414, + "loss": 0.3652667701244354, + "memory(GiB)": 41.24, + "nll_loss": 0.16899552941322327, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.050895966589450836, + "rewards/margins": 4.933393478393555, + "rewards/rejected": -4.882497310638428, + "step": 32, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.19027027027027027, + "grad_norm": 1.3124663829803467, + "learning_rate": 0.00019990052819973642, + "logits/chosen": -0.3465266227722168, + "logits/rejected": -0.6685288548469543, + "logps/chosen": -3.301164388656616, + "logps/rejected": -85.9378662109375, + "loss": 0.3046538233757019, + "memory(GiB)": 41.24, + "nll_loss": 0.1670866757631302, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17874348163604736, + "rewards/margins": 6.709518909454346, + "rewards/rejected": -6.530775547027588, + "step": 33, + "train_speed(iter/s)": 0.010748 + }, + { + "epoch": 0.19603603603603603, + "grad_norm": 2.341381549835205, + "learning_rate": 0.00019987008424349225, + "logits/chosen": -0.38037875294685364, + "logits/rejected": -0.699614405632019, + "logps/chosen": -1.5970546007156372, + "logps/rejected": -74.28025817871094, + "loss": 0.20820346474647522, + "memory(GiB)": 41.24, + "nll_loss": 0.10679290443658829, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11616384983062744, + "rewards/margins": 5.825808048248291, + "rewards/rejected": -5.709644794464111, + "step": 34, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.2018018018018018, + "grad_norm": 5.144662380218506, + "learning_rate": 0.00019983558482975799, + "logits/chosen": -0.46614986658096313, + "logits/rejected": -0.6599275469779968, + "logps/chosen": -5.982501983642578, + "logps/rejected": -67.7139892578125, + "loss": 0.5879941582679749, + "memory(GiB)": 41.24, + "nll_loss": 0.4303022027015686, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16055577993392944, + "rewards/margins": 5.052016258239746, + "rewards/rejected": -4.89146089553833, + "step": 35, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.20756756756756756, + "grad_norm": 1.990370273590088, + "learning_rate": 0.00019979703135946278, + "logits/chosen": -0.4655105471611023, + "logits/rejected": -0.6668959856033325, + "logps/chosen": -4.062808036804199, + "logps/rejected": -72.87210845947266, + "loss": 0.4453062415122986, + "memory(GiB)": 41.24, + "nll_loss": 0.25426411628723145, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.09591292589902878, + "rewards/margins": 5.528655052185059, + "rewards/rejected": -5.432741641998291, + "step": 36, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 0.21333333333333335, + "grad_norm": 2.0748369693756104, + "learning_rate": 0.00019975442539816012, + "logits/chosen": -0.4139191210269928, + "logits/rejected": -0.7536109685897827, + "logps/chosen": -5.956673622131348, + "logps/rejected": -66.99488830566406, + "loss": 0.5514004826545715, + "memory(GiB)": 41.24, + "nll_loss": 0.3969358503818512, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.33245787024497986, + "rewards/margins": 5.0754570960998535, + "rewards/rejected": -4.742999076843262, + "step": 37, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.2190990990990991, + "grad_norm": 1.6668081283569336, + "learning_rate": 0.0001997077686759643, + "logits/chosen": -0.42306363582611084, + "logits/rejected": -0.6915761232376099, + "logps/chosen": -3.814265727996826, + "logps/rejected": -82.30941772460938, + "loss": 0.3710038363933563, + "memory(GiB)": 41.24, + "nll_loss": 0.22380009293556213, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06087161600589752, + "rewards/margins": 6.076653957366943, + "rewards/rejected": -6.015782356262207, + "step": 38, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.22486486486486487, + "grad_norm": 13.827637672424316, + "learning_rate": 0.00019965706308748028, + "logits/chosen": -0.49769100546836853, + "logits/rejected": -0.7370949983596802, + "logps/chosen": -3.956310272216797, + "logps/rejected": -79.41323852539062, + "loss": 0.6633445620536804, + "memory(GiB)": 41.24, + "nll_loss": 0.3870386779308319, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.13494591414928436, + "rewards/margins": 6.147372722625732, + "rewards/rejected": -6.282319068908691, + "step": 39, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 0.23063063063063063, + "grad_norm": 2.19854474067688, + "learning_rate": 0.0001996023106917267, + "logits/chosen": -0.44343504309654236, + "logits/rejected": -0.6730125546455383, + "logps/chosen": -2.9238576889038086, + "logps/rejected": -77.40494537353516, + "loss": 0.3453696668148041, + "memory(GiB)": 41.24, + "nll_loss": 0.19069285690784454, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14595907926559448, + "rewards/margins": 6.002254486083984, + "rewards/rejected": -5.856295108795166, + "step": 40, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.2363963963963964, + "grad_norm": 6.0794596672058105, + "learning_rate": 0.00019954351371205203, + "logits/chosen": -0.47579658031463623, + "logits/rejected": -0.6826990842819214, + "logps/chosen": -5.335310935974121, + "logps/rejected": -75.78384399414062, + "loss": 0.575509250164032, + "memory(GiB)": 41.24, + "nll_loss": 0.4477939307689667, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2103371024131775, + "rewards/margins": 6.012211322784424, + "rewards/rejected": -5.801875114440918, + "step": 41, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.24216216216216216, + "grad_norm": 1.4152264595031738, + "learning_rate": 0.00019948067453604473, + "logits/chosen": -0.49620914459228516, + "logits/rejected": -0.7446714043617249, + "logps/chosen": -3.284031867980957, + "logps/rejected": -77.9183349609375, + "loss": 0.29828324913978577, + "memory(GiB)": 41.24, + "nll_loss": 0.21160344779491425, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18678154051303864, + "rewards/margins": 6.247434616088867, + "rewards/rejected": -6.060652732849121, + "step": 42, + "train_speed(iter/s)": 0.010755 + }, + { + "epoch": 0.24792792792792792, + "grad_norm": 2.9591963291168213, + "learning_rate": 0.00019941379571543596, + "logits/chosen": -0.4777877628803253, + "logits/rejected": -0.6974171996116638, + "logps/chosen": -2.9466118812561035, + "logps/rejected": -66.2719497680664, + "loss": 0.35521066188812256, + "memory(GiB)": 41.24, + "nll_loss": 0.22793087363243103, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.27246373891830444, + "rewards/margins": 5.403186798095703, + "rewards/rejected": -5.130722999572754, + "step": 43, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.2536936936936937, + "grad_norm": 4.199843883514404, + "learning_rate": 0.00019934287996599607, + "logits/chosen": -0.4372207522392273, + "logits/rejected": -0.700534462928772, + "logps/chosen": -3.31451416015625, + "logps/rejected": -70.0909194946289, + "loss": 0.3793654143810272, + "memory(GiB)": 41.24, + "nll_loss": 0.20433279871940613, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.059201404452323914, + "rewards/margins": 4.912087917327881, + "rewards/rejected": -4.852885723114014, + "step": 44, + "train_speed(iter/s)": 0.010756 + }, + { + "epoch": 0.2594594594594595, + "grad_norm": 0.7397028803825378, + "learning_rate": 0.00019926793016742435, + "logits/chosen": -0.42062634229660034, + "logits/rejected": -0.7593384981155396, + "logps/chosen": -1.404799222946167, + "logps/rejected": -102.4892349243164, + "loss": 0.1313902586698532, + "memory(GiB)": 41.24, + "nll_loss": 0.09521690011024475, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12229674309492111, + "rewards/margins": 8.041703224182129, + "rewards/rejected": -7.919405937194824, + "step": 45, + "train_speed(iter/s)": 0.010754 + }, + { + "epoch": 0.26522522522522524, + "grad_norm": 5.14201545715332, + "learning_rate": 0.00019918894936323195, + "logits/chosen": -0.3798273205757141, + "logits/rejected": -0.5865642428398132, + "logps/chosen": -6.911247730255127, + "logps/rejected": -72.06941223144531, + "loss": 0.49299517273902893, + "memory(GiB)": 41.24, + "nll_loss": 0.27924880385398865, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.1307646483182907, + "rewards/margins": 5.647462368011475, + "rewards/rejected": -5.516696929931641, + "step": 46, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.270990990990991, + "grad_norm": 1.6439752578735352, + "learning_rate": 0.00019910594076061853, + "logits/chosen": -0.4431299567222595, + "logits/rejected": -0.7268660664558411, + "logps/chosen": -2.0275888442993164, + "logps/rejected": -81.84529876708984, + "loss": 0.23740296065807343, + "memory(GiB)": 41.24, + "nll_loss": 0.14832313358783722, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06923090666532516, + "rewards/margins": 6.1783928871154785, + "rewards/rejected": -6.109161376953125, + "step": 47, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.27675675675675676, + "grad_norm": 6.499570846557617, + "learning_rate": 0.0001990189077303418, + "logits/chosen": -0.45142635703086853, + "logits/rejected": -0.7249264121055603, + "logps/chosen": -5.794652938842773, + "logps/rejected": -78.53363800048828, + "loss": 0.46158841252326965, + "memory(GiB)": 41.24, + "nll_loss": 0.3398270010948181, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24555836617946625, + "rewards/margins": 6.308343887329102, + "rewards/rejected": -6.062786102294922, + "step": 48, + "train_speed(iter/s)": 0.010753 + }, + { + "epoch": 0.2825225225225225, + "grad_norm": 5.58511209487915, + "learning_rate": 0.00019892785380658078, + "logits/chosen": -0.4099079370498657, + "logits/rejected": -0.6898340582847595, + "logps/chosen": -7.449429035186768, + "logps/rejected": -92.18098449707031, + "loss": 0.5815075039863586, + "memory(GiB)": 41.24, + "nll_loss": 0.41358357667922974, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.053265977650880814, + "rewards/margins": 7.150655269622803, + "rewards/rejected": -7.203920841217041, + "step": 49, + "train_speed(iter/s)": 0.010752 + }, + { + "epoch": 0.2882882882882883, + "grad_norm": 1.5411535501480103, + "learning_rate": 0.00019883278268679216, + "logits/chosen": -0.4670988917350769, + "logits/rejected": -0.6586328148841858, + "logps/chosen": -6.984646320343018, + "logps/rejected": -89.9208755493164, + "loss": 0.45735102891921997, + "memory(GiB)": 41.24, + "nll_loss": 0.28862154483795166, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1682412177324295, + "rewards/margins": 7.457679748535156, + "rewards/rejected": -7.289438247680664, + "step": 50, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 0.29405405405405405, + "grad_norm": 6.264907360076904, + "learning_rate": 0.00019873369823156036, + "logits/chosen": -0.35443419218063354, + "logits/rejected": -0.7265148758888245, + "logps/chosen": -5.11212158203125, + "logps/rejected": -97.37500762939453, + "loss": 0.648152232170105, + "memory(GiB)": 41.24, + "nll_loss": 0.5091428756713867, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.03551238775253296, + "rewards/margins": 7.55380392074585, + "rewards/rejected": -7.589317321777344, + "step": 51, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 0.2998198198198198, + "grad_norm": 2.1848013401031494, + "learning_rate": 0.00019863060446444054, + "logits/chosen": -0.5179776549339294, + "logits/rejected": -0.7255269289016724, + "logps/chosen": -6.782817363739014, + "logps/rejected": -86.8519515991211, + "loss": 0.5702924728393555, + "memory(GiB)": 41.24, + "nll_loss": 0.3926149308681488, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.05346503108739853, + "rewards/margins": 7.021766185760498, + "rewards/rejected": -6.968300819396973, + "step": 52, + "train_speed(iter/s)": 0.010708 + }, + { + "epoch": 0.3055855855855856, + "grad_norm": 3.3478569984436035, + "learning_rate": 0.00019852350557179538, + "logits/chosen": -0.5434097051620483, + "logits/rejected": -0.8009167313575745, + "logps/chosen": -4.053906440734863, + "logps/rejected": -105.9520263671875, + "loss": 0.4421282112598419, + "memory(GiB)": 41.24, + "nll_loss": 0.3492012917995453, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1857108175754547, + "rewards/margins": 9.035101890563965, + "rewards/rejected": -8.849390983581543, + "step": 53, + "train_speed(iter/s)": 0.010709 + }, + { + "epoch": 0.31135135135135134, + "grad_norm": 0.8761724829673767, + "learning_rate": 0.00019841240590262493, + "logits/chosen": -0.45293307304382324, + "logits/rejected": -0.7608242034912109, + "logps/chosen": -0.6350191831588745, + "logps/rejected": -91.82192993164062, + "loss": 0.11042662709951401, + "memory(GiB)": 41.24, + "nll_loss": 0.06284812837839127, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1965743899345398, + "rewards/margins": 7.520632743835449, + "rewards/rejected": -7.324058532714844, + "step": 54, + "train_speed(iter/s)": 0.01071 + }, + { + "epoch": 0.3171171171171171, + "grad_norm": 5.344722747802734, + "learning_rate": 0.0001982973099683902, + "logits/chosen": -0.5431379675865173, + "logits/rejected": -0.8100754022598267, + "logps/chosen": -1.98147714138031, + "logps/rejected": -91.00532531738281, + "loss": 0.3004758059978485, + "memory(GiB)": 41.24, + "nll_loss": 0.1778518110513687, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1774783581495285, + "rewards/margins": 7.744819641113281, + "rewards/rejected": -7.567340850830078, + "step": 55, + "train_speed(iter/s)": 0.010711 + }, + { + "epoch": 0.32288288288288286, + "grad_norm": 5.424385070800781, + "learning_rate": 0.00019817822244282973, + "logits/chosen": -0.48769453167915344, + "logits/rejected": -0.7255769371986389, + "logps/chosen": -3.6506757736206055, + "logps/rejected": -84.81509399414062, + "loss": 0.4457959830760956, + "memory(GiB)": 41.24, + "nll_loss": 0.23988068103790283, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.11002250760793686, + "rewards/margins": 6.652472019195557, + "rewards/rejected": -6.542449951171875, + "step": 56, + "train_speed(iter/s)": 0.010711 + }, + { + "epoch": 0.3286486486486486, + "grad_norm": 2.3812549114227295, + "learning_rate": 0.00019805514816177006, + "logits/chosen": -0.556337833404541, + "logits/rejected": -0.7923259735107422, + "logps/chosen": -5.455251693725586, + "logps/rejected": -89.06196594238281, + "loss": 0.4395422339439392, + "memory(GiB)": 41.24, + "nll_loss": 0.29659417271614075, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.01958906650543213, + "rewards/margins": 6.816771984100342, + "rewards/rejected": -6.836360931396484, + "step": 57, + "train_speed(iter/s)": 0.01071 + }, + { + "epoch": 0.33441441441441444, + "grad_norm": 8.72442626953125, + "learning_rate": 0.00019792809212292912, + "logits/chosen": -0.6302639842033386, + "logits/rejected": -0.9231972098350525, + "logps/chosen": -5.753312110900879, + "logps/rejected": -103.71651458740234, + "loss": 1.1447618007659912, + "memory(GiB)": 41.24, + "nll_loss": 0.6733683943748474, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.26626041531562805, + "rewards/margins": 8.245807647705078, + "rewards/rejected": -8.512067794799805, + "step": 58, + "train_speed(iter/s)": 0.010711 + }, + { + "epoch": 0.3401801801801802, + "grad_norm": 2.6023576259613037, + "learning_rate": 0.00019779705948571346, + "logits/chosen": -0.5867213010787964, + "logits/rejected": -0.8538931608200073, + "logps/chosen": -3.069394111633301, + "logps/rejected": -96.5278091430664, + "loss": 0.3167837858200073, + "memory(GiB)": 41.24, + "nll_loss": 0.20217999815940857, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10776210576295853, + "rewards/margins": 8.32030963897705, + "rewards/rejected": -8.21254825592041, + "step": 59, + "train_speed(iter/s)": 0.010712 + }, + { + "epoch": 0.34594594594594597, + "grad_norm": 2.678529977798462, + "learning_rate": 0.00019766205557100868, + "logits/chosen": -0.6138476133346558, + "logits/rejected": -0.7842435836791992, + "logps/chosen": -6.119746208190918, + "logps/rejected": -88.4565658569336, + "loss": 0.44386592507362366, + "memory(GiB)": 41.24, + "nll_loss": 0.2780015468597412, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06730165332555771, + "rewards/margins": 7.516484260559082, + "rewards/rejected": -7.449182987213135, + "step": 60, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.3517117117117117, + "grad_norm": 2.223165512084961, + "learning_rate": 0.00019752308586096326, + "logits/chosen": -0.5360291004180908, + "logits/rejected": -0.894256591796875, + "logps/chosen": -1.3611795902252197, + "logps/rejected": -122.1118392944336, + "loss": 0.13098187744617462, + "memory(GiB)": 41.24, + "nll_loss": 0.07775433361530304, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12457617372274399, + "rewards/margins": 10.008496284484863, + "rewards/rejected": -9.883918762207031, + "step": 61, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.3574774774774775, + "grad_norm": 10.891358375549316, + "learning_rate": 0.0001973801559987661, + "logits/chosen": -0.7173175811767578, + "logits/rejected": -0.8792683482170105, + "logps/chosen": -12.6211519241333, + "logps/rejected": -73.51658630371094, + "loss": 1.0753252506256104, + "memory(GiB)": 41.24, + "nll_loss": 0.9100903868675232, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.495037317276001, + "rewards/margins": 5.504097938537598, + "rewards/rejected": -5.9991350173950195, + "step": 62, + "train_speed(iter/s)": 0.010715 + }, + { + "epoch": 0.36324324324324325, + "grad_norm": 26.432544708251953, + "learning_rate": 0.00019723327178841743, + "logits/chosen": -0.6629639267921448, + "logits/rejected": -0.8958379030227661, + "logps/chosen": -5.678650856018066, + "logps/rejected": -81.07159423828125, + "loss": 0.6158818602561951, + "memory(GiB)": 41.24, + "nll_loss": 0.4310333728790283, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.02597002126276493, + "rewards/margins": 6.523518085479736, + "rewards/rejected": -6.497547149658203, + "step": 63, + "train_speed(iter/s)": 0.010716 + }, + { + "epoch": 0.369009009009009, + "grad_norm": 4.643962860107422, + "learning_rate": 0.0001970824391944927, + "logits/chosen": -0.6061327457427979, + "logits/rejected": -0.8394734859466553, + "logps/chosen": -6.52767276763916, + "logps/rejected": -74.48380279541016, + "loss": 0.5795359015464783, + "memory(GiB)": 41.24, + "nll_loss": 0.414874792098999, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0007130205631256104, + "rewards/margins": 5.6362080574035645, + "rewards/rejected": -5.635494709014893, + "step": 64, + "train_speed(iter/s)": 0.010716 + }, + { + "epoch": 0.3747747747747748, + "grad_norm": 1.8694281578063965, + "learning_rate": 0.00019692766434190105, + "logits/chosen": -0.6322288513183594, + "logits/rejected": -0.8566944003105164, + "logps/chosen": -1.9187999963760376, + "logps/rejected": -56.726871490478516, + "loss": 0.26561087369918823, + "memory(GiB)": 41.24, + "nll_loss": 0.12671466171741486, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.17268410325050354, + "rewards/margins": 4.417227268218994, + "rewards/rejected": -4.244543075561523, + "step": 65, + "train_speed(iter/s)": 0.010717 + }, + { + "epoch": 0.38054054054054054, + "grad_norm": 2.3087944984436035, + "learning_rate": 0.00019676895351563588, + "logits/chosen": -0.6048943996429443, + "logits/rejected": -0.8088940382003784, + "logps/chosen": -4.430207252502441, + "logps/rejected": -64.05681610107422, + "loss": 0.33754763007164, + "memory(GiB)": 41.24, + "nll_loss": 0.22767014801502228, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.025299668312072754, + "rewards/margins": 4.6934709548950195, + "rewards/rejected": -4.668170928955078, + "step": 66, + "train_speed(iter/s)": 0.010717 + }, + { + "epoch": 0.3863063063063063, + "grad_norm": 2.2111432552337646, + "learning_rate": 0.00019660631316052021, + "logits/chosen": -0.623546838760376, + "logits/rejected": -0.811290979385376, + "logps/chosen": -6.471018314361572, + "logps/rejected": -57.678314208984375, + "loss": 0.4503980875015259, + "memory(GiB)": 41.24, + "nll_loss": 0.34320953488349915, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.06837636232376099, + "rewards/margins": 4.10473108291626, + "rewards/rejected": -4.173107624053955, + "step": 67, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 0.39207207207207206, + "grad_norm": 1.7001421451568604, + "learning_rate": 0.00019643974988094458, + "logits/chosen": -0.5895470380783081, + "logits/rejected": -0.7888103723526001, + "logps/chosen": -5.163002014160156, + "logps/rejected": -45.541908264160156, + "loss": 0.42964497208595276, + "memory(GiB)": 41.24, + "nll_loss": 0.3103106617927551, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3456801772117615, + "rewards/margins": 3.432896614074707, + "rewards/rejected": -3.08721661567688, + "step": 68, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.3978378378378378, + "grad_norm": 3.909360885620117, + "learning_rate": 0.00019626927044059914, + "logits/chosen": -0.6134467124938965, + "logits/rejected": -0.7660009860992432, + "logps/chosen": -5.3520307540893555, + "logps/rejected": -49.259185791015625, + "loss": 0.5181165337562561, + "memory(GiB)": 41.24, + "nll_loss": 0.30421963334083557, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.014049924910068512, + "rewards/margins": 3.3693478107452393, + "rewards/rejected": -3.355297565460205, + "step": 69, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.4036036036036036, + "grad_norm": 2.67299485206604, + "learning_rate": 0.00019609488176219886, + "logits/chosen": -0.6107767820358276, + "logits/rejected": -0.7868641018867493, + "logps/chosen": -3.616837501525879, + "logps/rejected": -50.11117172241211, + "loss": 0.3146559000015259, + "memory(GiB)": 41.24, + "nll_loss": 0.21267317235469818, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13358786702156067, + "rewards/margins": 3.8761374950408936, + "rewards/rejected": -3.742549419403076, + "step": 70, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.40936936936936935, + "grad_norm": 7.013823509216309, + "learning_rate": 0.00019591659092720227, + "logits/chosen": -0.5896518230438232, + "logits/rejected": -0.858326256275177, + "logps/chosen": -2.8418002128601074, + "logps/rejected": -64.546630859375, + "loss": 0.37414392828941345, + "memory(GiB)": 41.24, + "nll_loss": 0.25448697805404663, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.024522263556718826, + "rewards/margins": 4.37087345123291, + "rewards/rejected": -4.346351623535156, + "step": 71, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 0.4151351351351351, + "grad_norm": 2.0888102054595947, + "learning_rate": 0.00019573440517552427, + "logits/chosen": -0.5669695138931274, + "logits/rejected": -0.7572087049484253, + "logps/chosen": -2.418893337249756, + "logps/rejected": -50.858699798583984, + "loss": 0.3724361062049866, + "memory(GiB)": 41.24, + "nll_loss": 0.20290419459342957, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.16552519798278809, + "rewards/margins": 3.5489120483398438, + "rewards/rejected": -3.3833868503570557, + "step": 72, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 0.4209009009009009, + "grad_norm": 1.9631896018981934, + "learning_rate": 0.00019554833190524182, + "logits/chosen": -0.575507402420044, + "logits/rejected": -0.7697166204452515, + "logps/chosen": -3.5829074382781982, + "logps/rejected": -52.79640197753906, + "loss": 0.400629460811615, + "memory(GiB)": 41.24, + "nll_loss": 0.2976799011230469, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17720367014408112, + "rewards/margins": 4.019702434539795, + "rewards/rejected": -3.842499017715454, + "step": 73, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.4266666666666667, + "grad_norm": 3.227459192276001, + "learning_rate": 0.00019535837867229363, + "logits/chosen": -0.6866841316223145, + "logits/rejected": -0.8467439413070679, + "logps/chosen": -1.4898688793182373, + "logps/rejected": -52.35246276855469, + "loss": 0.2691449224948883, + "memory(GiB)": 41.24, + "nll_loss": 0.16896064579486847, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20350010693073273, + "rewards/margins": 4.1118974685668945, + "rewards/rejected": -3.9083974361419678, + "step": 74, + "train_speed(iter/s)": 0.010726 + }, + { + "epoch": 0.43243243243243246, + "grad_norm": 2.0599851608276367, + "learning_rate": 0.0001951645531901734, + "logits/chosen": -0.6125411987304688, + "logits/rejected": -0.8178025484085083, + "logps/chosen": -5.753840923309326, + "logps/rejected": -60.490638732910156, + "loss": 0.4720686376094818, + "memory(GiB)": 41.24, + "nll_loss": 0.2955226004123688, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.02442253567278385, + "rewards/margins": 4.439631462097168, + "rewards/rejected": -4.41520881652832, + "step": 75, + "train_speed(iter/s)": 0.010726 + }, + { + "epoch": 0.4381981981981982, + "grad_norm": 1.8315515518188477, + "learning_rate": 0.00019496686332961646, + "logits/chosen": -0.5511488914489746, + "logits/rejected": -0.8103834390640259, + "logps/chosen": -2.737548828125, + "logps/rejected": -72.79937744140625, + "loss": 0.2598808705806732, + "memory(GiB)": 41.24, + "nll_loss": 0.20735348761081696, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20416659116744995, + "rewards/margins": 5.639776706695557, + "rewards/rejected": -5.435609817504883, + "step": 76, + "train_speed(iter/s)": 0.010726 + }, + { + "epoch": 0.443963963963964, + "grad_norm": 4.658299922943115, + "learning_rate": 0.00019476531711828027, + "logits/chosen": -0.5621539354324341, + "logits/rejected": -0.817601203918457, + "logps/chosen": -2.554128646850586, + "logps/rejected": -75.40179443359375, + "loss": 0.29839468002319336, + "memory(GiB)": 41.24, + "nll_loss": 0.21096321940422058, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11831088364124298, + "rewards/margins": 5.797797679901123, + "rewards/rejected": -5.679486274719238, + "step": 77, + "train_speed(iter/s)": 0.010727 + }, + { + "epoch": 0.44972972972972974, + "grad_norm": 7.5920844078063965, + "learning_rate": 0.00019455992274041835, + "logits/chosen": -0.5446800589561462, + "logits/rejected": -0.7866684794425964, + "logps/chosen": -5.180241584777832, + "logps/rejected": -80.02595520019531, + "loss": 0.4992501735687256, + "memory(GiB)": 41.24, + "nll_loss": 0.4071677029132843, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.006200561299920082, + "rewards/margins": 6.354435920715332, + "rewards/rejected": -6.348235607147217, + "step": 78, + "train_speed(iter/s)": 0.010728 + }, + { + "epoch": 0.4554954954954955, + "grad_norm": 2.045180082321167, + "learning_rate": 0.00019435068853654807, + "logits/chosen": -0.5892366170883179, + "logits/rejected": -0.782485842704773, + "logps/chosen": -7.082699775695801, + "logps/rejected": -76.8997573852539, + "loss": 0.4684464931488037, + "memory(GiB)": 41.24, + "nll_loss": 0.38624832034111023, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.03203499689698219, + "rewards/margins": 5.84527587890625, + "rewards/rejected": -5.813241004943848, + "step": 79, + "train_speed(iter/s)": 0.010728 + }, + { + "epoch": 0.46126126126126127, + "grad_norm": 6.412923812866211, + "learning_rate": 0.00019413762300311182, + "logits/chosen": -0.6524518728256226, + "logits/rejected": -0.8889943957328796, + "logps/chosen": -6.470341682434082, + "logps/rejected": -86.74302673339844, + "loss": 0.7839629650115967, + "memory(GiB)": 41.24, + "nll_loss": 0.6858316659927368, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.10852594673633575, + "rewards/margins": 6.819857120513916, + "rewards/rejected": -6.928382873535156, + "step": 80, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 0.46702702702702703, + "grad_norm": 2.171276807785034, + "learning_rate": 0.00019392073479213213, + "logits/chosen": -0.5607702136039734, + "logits/rejected": -0.7560180425643921, + "logps/chosen": -6.120340347290039, + "logps/rejected": -81.5516128540039, + "loss": 0.575468897819519, + "memory(GiB)": 41.24, + "nll_loss": 0.3079308271408081, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.05391833186149597, + "rewards/margins": 5.951426029205322, + "rewards/rejected": -6.005344390869141, + "step": 81, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 0.4727927927927928, + "grad_norm": 1.5074808597564697, + "learning_rate": 0.0001937000327108601, + "logits/chosen": -0.5817635655403137, + "logits/rejected": -0.8122580051422119, + "logps/chosen": -3.3293583393096924, + "logps/rejected": -76.75574493408203, + "loss": 0.27197888493537903, + "memory(GiB)": 41.24, + "nll_loss": 0.1920165866613388, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.04472556710243225, + "rewards/margins": 6.159332275390625, + "rewards/rejected": -6.114606857299805, + "step": 82, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 0.47855855855855856, + "grad_norm": 8.971928596496582, + "learning_rate": 0.0001934755257214181, + "logits/chosen": -0.5557482838630676, + "logits/rejected": -0.791240930557251, + "logps/chosen": -5.363508701324463, + "logps/rejected": -71.66944122314453, + "loss": 0.8509294986724854, + "memory(GiB)": 41.24, + "nll_loss": 0.7254281640052795, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.1657000482082367, + "rewards/margins": 5.284771919250488, + "rewards/rejected": -5.450472354888916, + "step": 83, + "train_speed(iter/s)": 0.010731 + }, + { + "epoch": 0.4843243243243243, + "grad_norm": 3.7317609786987305, + "learning_rate": 0.00019324722294043558, + "logits/chosen": -0.49631816148757935, + "logits/rejected": -0.7238998413085938, + "logps/chosen": -5.0319366455078125, + "logps/rejected": -55.3072395324707, + "loss": 0.4805355668067932, + "memory(GiB)": 41.24, + "nll_loss": 0.30268925428390503, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.16919195652008057, + "rewards/margins": 4.183167934417725, + "rewards/rejected": -4.013975620269775, + "step": 84, + "train_speed(iter/s)": 0.010732 + }, + { + "epoch": 0.4900900900900901, + "grad_norm": 7.251592636108398, + "learning_rate": 0.00019301513363867895, + "logits/chosen": -0.4797047972679138, + "logits/rejected": -0.699076771736145, + "logps/chosen": -4.068089485168457, + "logps/rejected": -62.654762268066406, + "loss": 0.41215279698371887, + "memory(GiB)": 41.24, + "nll_loss": 0.3088461756706238, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13345223665237427, + "rewards/margins": 4.483903884887695, + "rewards/rejected": -4.350452423095703, + "step": 85, + "train_speed(iter/s)": 0.010732 + }, + { + "epoch": 0.49585585585585584, + "grad_norm": 1.6465262174606323, + "learning_rate": 0.0001927792672406751, + "logits/chosen": -0.4875030219554901, + "logits/rejected": -0.7307705879211426, + "logps/chosen": -2.494859218597412, + "logps/rejected": -57.72221374511719, + "loss": 0.31443703174591064, + "memory(GiB)": 41.24, + "nll_loss": 0.1957024335861206, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1970575600862503, + "rewards/margins": 4.039019584655762, + "rewards/rejected": -3.8419618606567383, + "step": 86, + "train_speed(iter/s)": 0.010732 + }, + { + "epoch": 0.5016216216216216, + "grad_norm": 2.362053394317627, + "learning_rate": 0.00019253963332432878, + "logits/chosen": -0.6053374409675598, + "logits/rejected": -0.7439757585525513, + "logps/chosen": -5.3451008796691895, + "logps/rejected": -47.488853454589844, + "loss": 0.4348059296607971, + "memory(GiB)": 41.24, + "nll_loss": 0.23809021711349487, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.019803959876298904, + "rewards/margins": 3.427215099334717, + "rewards/rejected": -3.447019338607788, + "step": 87, + "train_speed(iter/s)": 0.010733 + }, + { + "epoch": 0.5073873873873874, + "grad_norm": 2.1728515625, + "learning_rate": 0.00019229624162053356, + "logits/chosen": -0.5406431555747986, + "logits/rejected": -0.739841103553772, + "logps/chosen": -2.6260576248168945, + "logps/rejected": -56.7901496887207, + "loss": 0.4087463915348053, + "memory(GiB)": 41.24, + "nll_loss": 0.2899452745914459, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0745973214507103, + "rewards/margins": 4.15200662612915, + "rewards/rejected": -4.077408790588379, + "step": 88, + "train_speed(iter/s)": 0.010733 + }, + { + "epoch": 0.5131531531531531, + "grad_norm": 1.563063144683838, + "learning_rate": 0.00019204910201277672, + "logits/chosen": -0.5593241453170776, + "logits/rejected": -0.7506377100944519, + "logps/chosen": -3.1961171627044678, + "logps/rejected": -63.025291442871094, + "loss": 0.20274411141872406, + "memory(GiB)": 41.24, + "nll_loss": 0.15050008893013, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13150721788406372, + "rewards/margins": 4.959373950958252, + "rewards/rejected": -4.827866554260254, + "step": 89, + "train_speed(iter/s)": 0.010734 + }, + { + "epoch": 0.518918918918919, + "grad_norm": 1.8057938814163208, + "learning_rate": 0.0001917982245367379, + "logits/chosen": -0.5387026071548462, + "logits/rejected": -0.779965341091156, + "logps/chosen": -5.658284664154053, + "logps/rejected": -61.30659484863281, + "loss": 0.5316486358642578, + "memory(GiB)": 41.24, + "nll_loss": 0.4009394347667694, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2519141137599945, + "rewards/margins": 4.251716613769531, + "rewards/rejected": -3.9998018741607666, + "step": 90, + "train_speed(iter/s)": 0.010734 + }, + { + "epoch": 0.5246846846846847, + "grad_norm": 1.6844978332519531, + "learning_rate": 0.00019154361937988163, + "logits/chosen": -0.5544890761375427, + "logits/rejected": -0.7064109444618225, + "logps/chosen": -6.5527496337890625, + "logps/rejected": -65.5311050415039, + "loss": 0.38739749789237976, + "memory(GiB)": 41.24, + "nll_loss": 0.2913617193698883, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1942709982395172, + "rewards/margins": 5.296841621398926, + "rewards/rejected": -5.102570533752441, + "step": 91, + "train_speed(iter/s)": 0.010734 + }, + { + "epoch": 0.5304504504504505, + "grad_norm": 3.110102653503418, + "learning_rate": 0.00019128529688104364, + "logits/chosen": -0.5477601885795593, + "logits/rejected": -0.7957394123077393, + "logps/chosen": -2.064303398132324, + "logps/rejected": -68.15472412109375, + "loss": 0.3176850974559784, + "memory(GiB)": 41.24, + "nll_loss": 0.21008610725402832, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18838940560817719, + "rewards/margins": 5.544022560119629, + "rewards/rejected": -5.355632781982422, + "step": 92, + "train_speed(iter/s)": 0.010735 + }, + { + "epoch": 0.5362162162162162, + "grad_norm": 3.8029165267944336, + "learning_rate": 0.00019102326753001086, + "logits/chosen": -0.5357910394668579, + "logits/rejected": -0.7163031101226807, + "logps/chosen": -4.390456676483154, + "logps/rejected": -55.20818328857422, + "loss": 0.4618917405605316, + "memory(GiB)": 41.24, + "nll_loss": 0.33474794030189514, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.36951008439064026, + "rewards/margins": 4.2895612716674805, + "rewards/rejected": -3.920051336288452, + "step": 93, + "train_speed(iter/s)": 0.010735 + }, + { + "epoch": 0.541981981981982, + "grad_norm": 1.8191365003585815, + "learning_rate": 0.00019075754196709572, + "logits/chosen": -0.5523058176040649, + "logits/rejected": -0.7712856531143188, + "logps/chosen": -2.1930363178253174, + "logps/rejected": -66.22615814208984, + "loss": 0.28284692764282227, + "memory(GiB)": 41.24, + "nll_loss": 0.2161579132080078, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14300797879695892, + "rewards/margins": 5.302109718322754, + "rewards/rejected": -5.159101486206055, + "step": 94, + "train_speed(iter/s)": 0.010735 + }, + { + "epoch": 0.5477477477477477, + "grad_norm": 2.65755558013916, + "learning_rate": 0.00019048813098270387, + "logits/chosen": -0.49344098567962646, + "logits/rejected": -0.7781980633735657, + "logps/chosen": -3.545104742050171, + "logps/rejected": -78.23759460449219, + "loss": 0.3401697278022766, + "memory(GiB)": 41.24, + "nll_loss": 0.3010118901729584, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07626594603061676, + "rewards/margins": 5.846086025238037, + "rewards/rejected": -5.769820213317871, + "step": 95, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 0.5535135135135135, + "grad_norm": 1.6140971183776855, + "learning_rate": 0.0001902150455168962, + "logits/chosen": -0.4496663808822632, + "logits/rejected": -0.7431093454360962, + "logps/chosen": -2.8349695205688477, + "logps/rejected": -72.62895965576172, + "loss": 0.3297802209854126, + "memory(GiB)": 41.24, + "nll_loss": 0.2532678544521332, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.149441659450531, + "rewards/margins": 5.512978553771973, + "rewards/rejected": -5.363536357879639, + "step": 96, + "train_speed(iter/s)": 0.010735 + }, + { + "epoch": 0.5592792792792792, + "grad_norm": 2.243736743927002, + "learning_rate": 0.0001899382966589443, + "logits/chosen": -0.44952693581581116, + "logits/rejected": -0.7028639316558838, + "logps/chosen": -5.277416229248047, + "logps/rejected": -69.73493957519531, + "loss": 0.41486623883247375, + "memory(GiB)": 41.24, + "nll_loss": 0.25434425473213196, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13436982035636902, + "rewards/margins": 4.8991923332214355, + "rewards/rejected": -4.764822006225586, + "step": 97, + "train_speed(iter/s)": 0.010735 + }, + { + "epoch": 0.565045045045045, + "grad_norm": 4.0099101066589355, + "learning_rate": 0.0001896578956468805, + "logits/chosen": -0.5480562448501587, + "logits/rejected": -0.7537853717803955, + "logps/chosen": -4.075699806213379, + "logps/rejected": -61.6773681640625, + "loss": 0.34289711713790894, + "memory(GiB)": 41.24, + "nll_loss": 0.24206680059432983, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09123305231332779, + "rewards/margins": 4.509951114654541, + "rewards/rejected": -4.418718338012695, + "step": 98, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 0.5708108108108109, + "grad_norm": 12.38167667388916, + "learning_rate": 0.00018937385386704126, + "logits/chosen": -0.6038029789924622, + "logits/rejected": -0.7428975105285645, + "logps/chosen": -6.938250541687012, + "logps/rejected": -58.5086555480957, + "loss": 0.4984804391860962, + "memory(GiB)": 41.24, + "nll_loss": 0.37261444330215454, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10895882546901703, + "rewards/margins": 4.678299903869629, + "rewards/rejected": -4.569341659545898, + "step": 99, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 0.5765765765765766, + "grad_norm": 4.255463123321533, + "learning_rate": 0.00018908618285360494, + "logits/chosen": -0.5780411958694458, + "logits/rejected": -0.7493711709976196, + "logps/chosen": -6.087734222412109, + "logps/rejected": -68.05068969726562, + "loss": 0.5377764701843262, + "memory(GiB)": 41.24, + "nll_loss": 0.4144035577774048, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07401561737060547, + "rewards/margins": 5.022759437561035, + "rewards/rejected": -4.94874382019043, + "step": 100, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 0.5823423423423424, + "grad_norm": 2.052121162414551, + "learning_rate": 0.00018879489428812334, + "logits/chosen": -0.5155804753303528, + "logits/rejected": -0.7454657554626465, + "logps/chosen": -4.093721389770508, + "logps/rejected": -73.28987121582031, + "loss": 0.4639092683792114, + "memory(GiB)": 41.24, + "nll_loss": 0.3733065724372864, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.16241005063056946, + "rewards/margins": 5.644482135772705, + "rewards/rejected": -5.482071876525879, + "step": 101, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 0.5881081081081081, + "grad_norm": 2.1512622833251953, + "learning_rate": 0.0001884999999990475, + "logits/chosen": -0.5633401274681091, + "logits/rejected": -0.7381998896598816, + "logps/chosen": -6.3428497314453125, + "logps/rejected": -52.1317138671875, + "loss": 0.4867536425590515, + "memory(GiB)": 41.24, + "nll_loss": 0.34330999851226807, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.33921706676483154, + "rewards/margins": 4.20556116104126, + "rewards/rejected": -3.8663439750671387, + "step": 102, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 0.5938738738738739, + "grad_norm": 3.3250627517700195, + "learning_rate": 0.00018820151196124717, + "logits/chosen": -0.5157067179679871, + "logits/rejected": -0.8171908855438232, + "logps/chosen": -3.4731268882751465, + "logps/rejected": -87.99000549316406, + "loss": 0.5975285768508911, + "memory(GiB)": 41.24, + "nll_loss": 0.46800604462623596, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06358014047145844, + "rewards/margins": 6.483148097991943, + "rewards/rejected": -6.419568061828613, + "step": 103, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 0.5996396396396396, + "grad_norm": 3.476541757583618, + "learning_rate": 0.0001878994422955246, + "logits/chosen": -0.5131168365478516, + "logits/rejected": -0.8059231042861938, + "logps/chosen": -3.1174309253692627, + "logps/rejected": -86.71981811523438, + "loss": 0.3160158693790436, + "memory(GiB)": 41.24, + "nll_loss": 0.25080835819244385, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.28578388690948486, + "rewards/margins": 6.907861232757568, + "rewards/rejected": -6.622077941894531, + "step": 104, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 0.6054054054054054, + "grad_norm": 5.32436990737915, + "learning_rate": 0.00018759380326812257, + "logits/chosen": -0.6340402960777283, + "logits/rejected": -0.867938756942749, + "logps/chosen": -2.323990821838379, + "logps/rejected": -71.03258514404297, + "loss": 0.31046488881111145, + "memory(GiB)": 41.24, + "nll_loss": 0.21423426270484924, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.011083951219916344, + "rewards/margins": 6.008091926574707, + "rewards/rejected": -6.0191755294799805, + "step": 105, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 0.6111711711711711, + "grad_norm": 3.4041733741760254, + "learning_rate": 0.0001872846072902259, + "logits/chosen": -0.589099645614624, + "logits/rejected": -0.8120458126068115, + "logps/chosen": -1.5791796445846558, + "logps/rejected": -82.34098052978516, + "loss": 0.2268940657377243, + "memory(GiB)": 41.24, + "nll_loss": 0.14617523550987244, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.2277921438217163, + "rewards/margins": 6.835321426391602, + "rewards/rejected": -6.607529640197754, + "step": 106, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 0.616936936936937, + "grad_norm": 1.9216675758361816, + "learning_rate": 0.00018697186691745782, + "logits/chosen": -0.599471926689148, + "logits/rejected": -0.8271054029464722, + "logps/chosen": -4.24713134765625, + "logps/rejected": -84.08229064941406, + "loss": 0.3841734826564789, + "memory(GiB)": 41.24, + "nll_loss": 0.31657740473747253, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.022511344403028488, + "rewards/margins": 6.769836902618408, + "rewards/rejected": -6.747325897216797, + "step": 107, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.6227027027027027, + "grad_norm": 4.744035720825195, + "learning_rate": 0.0001866555948493699, + "logits/chosen": -0.5849882960319519, + "logits/rejected": -0.8727071285247803, + "logps/chosen": -2.891716718673706, + "logps/rejected": -92.83207702636719, + "loss": 0.31115370988845825, + "memory(GiB)": 41.24, + "nll_loss": 0.22799423336982727, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19745981693267822, + "rewards/margins": 7.553801536560059, + "rewards/rejected": -7.35634183883667, + "step": 108, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.6284684684684685, + "grad_norm": 2.028069019317627, + "learning_rate": 0.00018633580392892648, + "logits/chosen": -0.5856168270111084, + "logits/rejected": -0.8363710641860962, + "logps/chosen": -5.515890121459961, + "logps/rejected": -83.518310546875, + "loss": 0.40675532817840576, + "memory(GiB)": 41.24, + "nll_loss": 0.33311402797698975, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.19537094235420227, + "rewards/margins": 6.882720947265625, + "rewards/rejected": -6.687350273132324, + "step": 109, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.6342342342342342, + "grad_norm": 1.8457213640213013, + "learning_rate": 0.00018601250714198302, + "logits/chosen": -0.6694027185440063, + "logits/rejected": -0.8495978713035583, + "logps/chosen": -6.527522087097168, + "logps/rejected": -69.24601745605469, + "loss": 0.4539996385574341, + "memory(GiB)": 41.24, + "nll_loss": 0.3218171000480652, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16864001750946045, + "rewards/margins": 5.4212541580200195, + "rewards/rejected": -5.252613544464111, + "step": 110, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.64, + "grad_norm": 1.4716756343841553, + "learning_rate": 0.00018568571761675893, + "logits/chosen": -0.5902320742607117, + "logits/rejected": -0.8840252757072449, + "logps/chosen": -5.05216121673584, + "logps/rejected": -96.3882064819336, + "loss": 0.3378267288208008, + "memory(GiB)": 41.24, + "nll_loss": 0.2499268501996994, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21043285727500916, + "rewards/margins": 7.197650909423828, + "rewards/rejected": -6.987217903137207, + "step": 111, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.6457657657657657, + "grad_norm": 0.7805799245834351, + "learning_rate": 0.00018535544862330436, + "logits/chosen": -0.6686023473739624, + "logits/rejected": -0.986074686050415, + "logps/chosen": -2.231717824935913, + "logps/rejected": -96.76434326171875, + "loss": 0.16498756408691406, + "memory(GiB)": 41.24, + "nll_loss": 0.14222995936870575, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14315873384475708, + "rewards/margins": 7.930060386657715, + "rewards/rejected": -7.786901473999023, + "step": 112, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 0.6515315315315315, + "grad_norm": 2.115647792816162, + "learning_rate": 0.00018502171357296144, + "logits/chosen": -0.7186330556869507, + "logits/rejected": -0.9198458194732666, + "logps/chosen": -6.138432502746582, + "logps/rejected": -76.40232849121094, + "loss": 0.42215695977211, + "memory(GiB)": 41.24, + "nll_loss": 0.34653419256210327, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3074129521846771, + "rewards/margins": 6.320471286773682, + "rewards/rejected": -6.013057708740234, + "step": 113, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.6572972972972972, + "grad_norm": 8.20527458190918, + "learning_rate": 0.00018468452601781948, + "logits/chosen": -0.6733801364898682, + "logits/rejected": -0.9577184319496155, + "logps/chosen": -5.523393154144287, + "logps/rejected": -94.21669006347656, + "loss": 0.4980089068412781, + "memory(GiB)": 41.24, + "nll_loss": 0.37918829917907715, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07570207118988037, + "rewards/margins": 7.661881446838379, + "rewards/rejected": -7.586179256439209, + "step": 114, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.6630630630630631, + "grad_norm": 1.4824076890945435, + "learning_rate": 0.00018434389965016495, + "logits/chosen": -0.6685976386070251, + "logits/rejected": -0.9235626459121704, + "logps/chosen": -4.8032307624816895, + "logps/rejected": -89.84734344482422, + "loss": 0.3988059163093567, + "memory(GiB)": 41.24, + "nll_loss": 0.3272039592266083, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1318884938955307, + "rewards/margins": 7.2766876220703125, + "rewards/rejected": -7.14479923248291, + "step": 115, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.6688288288288289, + "grad_norm": 6.457195281982422, + "learning_rate": 0.00018399984830192522, + "logits/chosen": -0.6563739776611328, + "logits/rejected": -0.9446012377738953, + "logps/chosen": -5.333343505859375, + "logps/rejected": -96.29754638671875, + "loss": 0.6648478507995605, + "memory(GiB)": 41.24, + "nll_loss": 0.5736638307571411, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1766398400068283, + "rewards/margins": 7.466494560241699, + "rewards/rejected": -7.289853572845459, + "step": 116, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.6745945945945946, + "grad_norm": 0.6062999367713928, + "learning_rate": 0.00018365238594410696, + "logits/chosen": -0.6826354265213013, + "logits/rejected": -0.9526041150093079, + "logps/chosen": -2.1560752391815186, + "logps/rejected": -97.08880615234375, + "loss": 0.15202483534812927, + "memory(GiB)": 41.24, + "nll_loss": 0.1097334772348404, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08674149215221405, + "rewards/margins": 8.09538745880127, + "rewards/rejected": -8.008646011352539, + "step": 117, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.6803603603603604, + "grad_norm": 4.866965293884277, + "learning_rate": 0.00018330152668622892, + "logits/chosen": -0.6270961761474609, + "logits/rejected": -0.8659616708755493, + "logps/chosen": -6.8050456047058105, + "logps/rejected": -94.24461364746094, + "loss": 0.5779068470001221, + "memory(GiB)": 41.24, + "nll_loss": 0.4634403884410858, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08972588181495667, + "rewards/margins": 7.540095806121826, + "rewards/rejected": -7.450369834899902, + "step": 118, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.6861261261261261, + "grad_norm": 1.9984689950942993, + "learning_rate": 0.00018294728477574886, + "logits/chosen": -0.6620824933052063, + "logits/rejected": -0.9279792308807373, + "logps/chosen": -3.62056303024292, + "logps/rejected": -88.79190826416016, + "loss": 0.5508708357810974, + "memory(GiB)": 41.24, + "nll_loss": 0.44049572944641113, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04914311319589615, + "rewards/margins": 7.090651512145996, + "rewards/rejected": -7.041508197784424, + "step": 119, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 0.6918918918918919, + "grad_norm": 2.634037971496582, + "learning_rate": 0.000182589674597485, + "logits/chosen": -0.6129943132400513, + "logits/rejected": -0.891259491443634, + "logps/chosen": -4.1637163162231445, + "logps/rejected": -89.52510833740234, + "loss": 0.5133303999900818, + "memory(GiB)": 41.24, + "nll_loss": 0.35086461901664734, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.015482299029827118, + "rewards/margins": 7.1094279289245605, + "rewards/rejected": -7.0939459800720215, + "step": 120, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 0.6976576576576576, + "grad_norm": 2.7602248191833496, + "learning_rate": 0.00018222871067303192, + "logits/chosen": -0.644217848777771, + "logits/rejected": -0.8804385662078857, + "logps/chosen": -4.665724277496338, + "logps/rejected": -75.18159484863281, + "loss": 0.4733228385448456, + "memory(GiB)": 41.24, + "nll_loss": 0.3284577429294586, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11940830945968628, + "rewards/margins": 6.475360870361328, + "rewards/rejected": -6.355952262878418, + "step": 121, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 0.7034234234234235, + "grad_norm": 8.460821151733398, + "learning_rate": 0.0001818644076601709, + "logits/chosen": -0.6995394825935364, + "logits/rejected": -0.9071613550186157, + "logps/chosen": -8.95744800567627, + "logps/rejected": -77.84011840820312, + "loss": 0.6963841915130615, + "memory(GiB)": 41.24, + "nll_loss": 0.5519821047782898, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.2606813609600067, + "rewards/margins": 6.603857040405273, + "rewards/rejected": -6.343176364898682, + "step": 122, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 0.7091891891891892, + "grad_norm": 3.587855100631714, + "learning_rate": 0.00018149678035227473, + "logits/chosen": -0.6442317962646484, + "logits/rejected": -0.8487176895141602, + "logps/chosen": -6.872650623321533, + "logps/rejected": -82.89649963378906, + "loss": 0.4728536605834961, + "memory(GiB)": 41.24, + "nll_loss": 0.3608010411262512, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.009775836020708084, + "rewards/margins": 6.596733093261719, + "rewards/rejected": -6.586956977844238, + "step": 123, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 0.714954954954955, + "grad_norm": 1.7244524955749512, + "learning_rate": 0.00018112584367770685, + "logits/chosen": -0.6685165762901306, + "logits/rejected": -0.8828461766242981, + "logps/chosen": -5.397648334503174, + "logps/rejected": -85.60098266601562, + "loss": 0.389848530292511, + "memory(GiB)": 41.24, + "nll_loss": 0.32140427827835083, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12320394814014435, + "rewards/margins": 7.233897686004639, + "rewards/rejected": -7.11069393157959, + "step": 124, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 0.7207207207207207, + "grad_norm": 0.7455915808677673, + "learning_rate": 0.00018075161269921527, + "logits/chosen": -0.5614687204360962, + "logits/rejected": -0.8907942771911621, + "logps/chosen": -1.7438545227050781, + "logps/rejected": -92.23735809326172, + "loss": 0.12226351350545883, + "memory(GiB)": 41.24, + "nll_loss": 0.10527773201465607, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15749390423297882, + "rewards/margins": 7.594954967498779, + "rewards/rejected": -7.437460422515869, + "step": 125, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 0.7264864864864865, + "grad_norm": 1.0758198499679565, + "learning_rate": 0.000180374102613321, + "logits/chosen": -0.6551331877708435, + "logits/rejected": -0.8264045119285583, + "logps/chosen": -4.308568000793457, + "logps/rejected": -80.06883239746094, + "loss": 0.36240947246551514, + "memory(GiB)": 41.24, + "nll_loss": 0.24730846285820007, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10061657428741455, + "rewards/margins": 6.429587364196777, + "rewards/rejected": -6.328970909118652, + "step": 126, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 0.7322522522522522, + "grad_norm": 1.0857983827590942, + "learning_rate": 0.0001799933287497007, + "logits/chosen": -0.6380553841590881, + "logits/rejected": -0.8699716925621033, + "logps/chosen": -1.9937247037887573, + "logps/rejected": -87.90292358398438, + "loss": 0.18343912065029144, + "memory(GiB)": 41.24, + "nll_loss": 0.12965701520442963, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09683795273303986, + "rewards/margins": 7.11926794052124, + "rewards/rejected": -7.022429943084717, + "step": 127, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.738018018018018, + "grad_norm": 9.539369583129883, + "learning_rate": 0.00017960930657056438, + "logits/chosen": -0.6350418925285339, + "logits/rejected": -0.8755402565002441, + "logps/chosen": -2.645111560821533, + "logps/rejected": -82.07209777832031, + "loss": 0.23406416177749634, + "memory(GiB)": 41.24, + "nll_loss": 0.17941246926784515, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07386763393878937, + "rewards/margins": 6.727149486541748, + "rewards/rejected": -6.653281211853027, + "step": 128, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.7437837837837837, + "grad_norm": 10.807381629943848, + "learning_rate": 0.00017922205167002754, + "logits/chosen": -0.6329447031021118, + "logits/rejected": -0.8397620320320129, + "logps/chosen": -4.6306023597717285, + "logps/rejected": -67.41867065429688, + "loss": 0.5849911570549011, + "memory(GiB)": 41.24, + "nll_loss": 0.41021403670310974, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06239153444766998, + "rewards/margins": 5.311315059661865, + "rewards/rejected": -5.248923301696777, + "step": 129, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.7495495495495496, + "grad_norm": 1.8455349206924438, + "learning_rate": 0.00017883157977347774, + "logits/chosen": -0.5277632474899292, + "logits/rejected": -0.785979151725769, + "logps/chosen": -4.069696426391602, + "logps/rejected": -82.98876953125, + "loss": 0.3647957146167755, + "memory(GiB)": 41.24, + "nll_loss": 0.24460868537425995, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.12994280457496643, + "rewards/margins": 6.137853622436523, + "rewards/rejected": -6.00791072845459, + "step": 130, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.7553153153153153, + "grad_norm": 1.5157482624053955, + "learning_rate": 0.00017843790673693625, + "logits/chosen": -0.533730685710907, + "logits/rejected": -0.8200556635856628, + "logps/chosen": -4.038061141967773, + "logps/rejected": -82.76172637939453, + "loss": 0.2888023555278778, + "memory(GiB)": 41.24, + "nll_loss": 0.2086402326822281, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2728549540042877, + "rewards/margins": 6.415480613708496, + "rewards/rejected": -6.142626762390137, + "step": 131, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.7610810810810811, + "grad_norm": 1.936601996421814, + "learning_rate": 0.00017804104854641408, + "logits/chosen": -0.6249516606330872, + "logits/rejected": -0.8721716403961182, + "logps/chosen": -3.304114818572998, + "logps/rejected": -60.27552795410156, + "loss": 0.415643572807312, + "memory(GiB)": 41.24, + "nll_loss": 0.28065580129623413, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2913169264793396, + "rewards/margins": 4.785688400268555, + "rewards/rejected": -4.49437141418457, + "step": 132, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.7668468468468469, + "grad_norm": 3.6359870433807373, + "learning_rate": 0.0001776410213172628, + "logits/chosen": -0.5627670288085938, + "logits/rejected": -0.8085115551948547, + "logps/chosen": -1.7719781398773193, + "logps/rejected": -72.61830139160156, + "loss": 0.2871711552143097, + "memory(GiB)": 41.24, + "nll_loss": 0.1900225728750229, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.089139424264431, + "rewards/margins": 5.8200178146362305, + "rewards/rejected": -5.730878829956055, + "step": 133, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.7726126126126126, + "grad_norm": 1.4323631525039673, + "learning_rate": 0.00017723784129352018, + "logits/chosen": -0.6231422424316406, + "logits/rejected": -0.8504309058189392, + "logps/chosen": -1.730849027633667, + "logps/rejected": -68.78044128417969, + "loss": 0.2242366373538971, + "memory(GiB)": 41.24, + "nll_loss": 0.13950544595718384, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18260075151920319, + "rewards/margins": 5.708943843841553, + "rewards/rejected": -5.526342391967773, + "step": 134, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.7783783783783784, + "grad_norm": 1.6163887977600098, + "learning_rate": 0.00017683152484725066, + "logits/chosen": -0.6224597692489624, + "logits/rejected": -0.7799749374389648, + "logps/chosen": -4.717617988586426, + "logps/rejected": -61.31199264526367, + "loss": 0.4397742450237274, + "memory(GiB)": 41.24, + "nll_loss": 0.3140389323234558, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.25990185141563416, + "rewards/margins": 4.902276039123535, + "rewards/rejected": -4.642374515533447, + "step": 135, + "train_speed(iter/s)": 0.010726 + }, + { + "epoch": 0.7841441441441441, + "grad_norm": 1.2988686561584473, + "learning_rate": 0.00017642208847788032, + "logits/chosen": -0.4455900192260742, + "logits/rejected": -0.8272579312324524, + "logps/chosen": -3.7604730129241943, + "logps/rejected": -91.75283813476562, + "loss": 0.3069111108779907, + "memory(GiB)": 41.24, + "nll_loss": 0.257954478263855, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17708764970302582, + "rewards/margins": 7.115184783935547, + "rewards/rejected": -6.93809700012207, + "step": 136, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.78990990990991, + "grad_norm": 3.960479974746704, + "learning_rate": 0.00017600954881152693, + "logits/chosen": -0.480897456407547, + "logits/rejected": -0.806812584400177, + "logps/chosen": -2.339344024658203, + "logps/rejected": -88.25721740722656, + "loss": 0.25455033779144287, + "memory(GiB)": 41.24, + "nll_loss": 0.1620534360408783, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10610076785087585, + "rewards/margins": 6.920160293579102, + "rewards/rejected": -6.814059734344482, + "step": 137, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.7956756756756757, + "grad_norm": 9.401166915893555, + "learning_rate": 0.00017559392260032506, + "logits/chosen": -0.4589274525642395, + "logits/rejected": -0.7634912729263306, + "logps/chosen": -5.8359503746032715, + "logps/rejected": -96.87335205078125, + "loss": 0.5270398855209351, + "memory(GiB)": 41.24, + "nll_loss": 0.43480122089385986, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.14989905059337616, + "rewards/margins": 7.5454607009887695, + "rewards/rejected": -7.69536018371582, + "step": 138, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.8014414414414415, + "grad_norm": 4.686446666717529, + "learning_rate": 0.00017517522672174548, + "logits/chosen": -0.5010412931442261, + "logits/rejected": -0.8063571453094482, + "logps/chosen": -2.5103189945220947, + "logps/rejected": -96.94129943847656, + "loss": 0.3778824508190155, + "memory(GiB)": 41.24, + "nll_loss": 0.252981573343277, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10596343874931335, + "rewards/margins": 7.605740070343018, + "rewards/rejected": -7.499776840209961, + "step": 139, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.8072072072072072, + "grad_norm": 1.6708407402038574, + "learning_rate": 0.00017475347817790996, + "logits/chosen": -0.4751094877719879, + "logits/rejected": -0.8238545060157776, + "logps/chosen": -3.2878451347351074, + "logps/rejected": -86.26036071777344, + "loss": 0.24279947578907013, + "memory(GiB)": 41.24, + "nll_loss": 0.16919009387493134, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07166648656129837, + "rewards/margins": 6.587783336639404, + "rewards/rejected": -6.516116142272949, + "step": 140, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.812972972972973, + "grad_norm": 1.4335037469863892, + "learning_rate": 0.00017432869409490093, + "logits/chosen": -0.5489400625228882, + "logits/rejected": -0.7731753587722778, + "logps/chosen": -5.348051071166992, + "logps/rejected": -70.64154815673828, + "loss": 0.36098673939704895, + "memory(GiB)": 41.24, + "nll_loss": 0.2713705897331238, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23904970288276672, + "rewards/margins": 5.921088695526123, + "rewards/rejected": -5.682039737701416, + "step": 141, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.8187387387387387, + "grad_norm": 2.012854814529419, + "learning_rate": 0.00017390089172206592, + "logits/chosen": -0.467896968126297, + "logits/rejected": -0.793892502784729, + "logps/chosen": -1.7580054998397827, + "logps/rejected": -85.476318359375, + "loss": 0.23942089080810547, + "memory(GiB)": 41.24, + "nll_loss": 0.17797623574733734, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09021879732608795, + "rewards/margins": 6.8085222244262695, + "rewards/rejected": -6.718303680419922, + "step": 142, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.8245045045045045, + "grad_norm": 5.257446765899658, + "learning_rate": 0.00017347008843131712, + "logits/chosen": -0.4858510196208954, + "logits/rejected": -0.7565414309501648, + "logps/chosen": -3.7341561317443848, + "logps/rejected": -75.44990539550781, + "loss": 0.43588101863861084, + "memory(GiB)": 41.24, + "nll_loss": 0.3324872553348541, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1589755266904831, + "rewards/margins": 5.903942584991455, + "rewards/rejected": -5.744967460632324, + "step": 143, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.8302702702702702, + "grad_norm": 1.9966129064559937, + "learning_rate": 0.00017303630171642607, + "logits/chosen": -0.4013749659061432, + "logits/rejected": -0.7098768353462219, + "logps/chosen": -2.0640015602111816, + "logps/rejected": -80.94207763671875, + "loss": 0.2669733166694641, + "memory(GiB)": 41.24, + "nll_loss": 0.2289275974035263, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20508992671966553, + "rewards/margins": 5.947912216186523, + "rewards/rejected": -5.742822170257568, + "step": 144, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 0.836036036036036, + "grad_norm": 4.144430637359619, + "learning_rate": 0.0001725995491923131, + "logits/chosen": -0.34412139654159546, + "logits/rejected": -0.7268174290657043, + "logps/chosen": -2.0844953060150146, + "logps/rejected": -86.29147338867188, + "loss": 0.2860412001609802, + "memory(GiB)": 41.24, + "nll_loss": 0.21140313148498535, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1814691722393036, + "rewards/margins": 6.325296401977539, + "rewards/rejected": -6.143826961517334, + "step": 145, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.8418018018018018, + "grad_norm": 6.901271343231201, + "learning_rate": 0.0001721598485943322, + "logits/chosen": -0.4293047785758972, + "logits/rejected": -0.6931589841842651, + "logps/chosen": -3.5573177337646484, + "logps/rejected": -80.00125885009766, + "loss": 0.402279257774353, + "memory(GiB)": 41.24, + "nll_loss": 0.30397528409957886, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.027988389134407043, + "rewards/margins": 6.108896732330322, + "rewards/rejected": -6.136884689331055, + "step": 146, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.8475675675675676, + "grad_norm": 1.3138624429702759, + "learning_rate": 0.00017171721777755074, + "logits/chosen": -0.441506952047348, + "logits/rejected": -0.6183236241340637, + "logps/chosen": -5.245753288269043, + "logps/rejected": -61.3995246887207, + "loss": 0.5000537633895874, + "memory(GiB)": 41.24, + "nll_loss": 0.3299512565135956, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1415078043937683, + "rewards/margins": 4.831845760345459, + "rewards/rejected": -4.690337181091309, + "step": 147, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.8533333333333334, + "grad_norm": 2.7394113540649414, + "learning_rate": 0.00017127167471602447, + "logits/chosen": -0.44564759731292725, + "logits/rejected": -0.7324830889701843, + "logps/chosen": -2.2436249256134033, + "logps/rejected": -73.5162124633789, + "loss": 0.20336386561393738, + "memory(GiB)": 41.24, + "nll_loss": 0.11223578453063965, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.17428822815418243, + "rewards/margins": 5.678185939788818, + "rewards/rejected": -5.503897666931152, + "step": 148, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.8590990990990991, + "grad_norm": 1.488072395324707, + "learning_rate": 0.00017082323750206761, + "logits/chosen": -0.44784775376319885, + "logits/rejected": -0.6909451484680176, + "logps/chosen": -3.5523109436035156, + "logps/rejected": -70.1702651977539, + "loss": 0.31512442231178284, + "memory(GiB)": 41.24, + "nll_loss": 0.2102314829826355, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11678414791822433, + "rewards/margins": 5.312015533447266, + "rewards/rejected": -5.1952314376831055, + "step": 149, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.8648648648648649, + "grad_norm": 2.6578073501586914, + "learning_rate": 0.00017037192434551823, + "logits/chosen": -0.39348986744880676, + "logits/rejected": -0.690170407295227, + "logps/chosen": -3.4318065643310547, + "logps/rejected": -76.70863342285156, + "loss": 0.40308576822280884, + "memory(GiB)": 41.24, + "nll_loss": 0.2762233316898346, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1851682811975479, + "rewards/margins": 5.912222385406494, + "rewards/rejected": -5.727054119110107, + "step": 150, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 0.8706306306306306, + "grad_norm": 2.9397995471954346, + "learning_rate": 0.00016991775357299866, + "logits/chosen": -0.4844151735305786, + "logits/rejected": -0.7689380645751953, + "logps/chosen": -4.010564804077148, + "logps/rejected": -85.45973205566406, + "loss": 0.3323879539966583, + "memory(GiB)": 41.24, + "nll_loss": 0.2697053551673889, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17146898806095123, + "rewards/margins": 6.537592887878418, + "rewards/rejected": -6.366123676300049, + "step": 151, + "train_speed(iter/s)": 0.010712 + }, + { + "epoch": 0.8763963963963964, + "grad_norm": 2.216646909713745, + "learning_rate": 0.00016946074362717147, + "logits/chosen": -0.5326839685440063, + "logits/rejected": -0.7718309760093689, + "logps/chosen": -4.843362808227539, + "logps/rejected": -73.73052978515625, + "loss": 0.4268698990345001, + "memory(GiB)": 41.24, + "nll_loss": 0.36223095655441284, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21485474705696106, + "rewards/margins": 5.929214000701904, + "rewards/rejected": -5.714358329772949, + "step": 152, + "train_speed(iter/s)": 0.010712 + }, + { + "epoch": 0.8821621621621621, + "grad_norm": 6.382073879241943, + "learning_rate": 0.00016900091306599042, + "logits/chosen": -0.6236099004745483, + "logits/rejected": -0.7957028150558472, + "logps/chosen": -6.768365859985352, + "logps/rejected": -75.18216705322266, + "loss": 0.5070593953132629, + "memory(GiB)": 41.24, + "nll_loss": 0.39859798550605774, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.03324725478887558, + "rewards/margins": 6.097735404968262, + "rewards/rejected": -6.130982875823975, + "step": 153, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.887927927927928, + "grad_norm": 5.002453327178955, + "learning_rate": 0.00016853828056194697, + "logits/chosen": -0.47666168212890625, + "logits/rejected": -0.7901638150215149, + "logps/chosen": -1.6101099252700806, + "logps/rejected": -83.95780944824219, + "loss": 0.2384922355413437, + "memory(GiB)": 41.24, + "nll_loss": 0.15521878004074097, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.027309579774737358, + "rewards/margins": 6.557631492614746, + "rewards/rejected": -6.530322074890137, + "step": 154, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.8936936936936937, + "grad_norm": 2.2284231185913086, + "learning_rate": 0.00016807286490131196, + "logits/chosen": -0.4581748843193054, + "logits/rejected": -0.7216594815254211, + "logps/chosen": -4.746794700622559, + "logps/rejected": -65.14288330078125, + "loss": 0.4944155812263489, + "memory(GiB)": 41.24, + "nll_loss": 0.38365647196769714, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09537286311388016, + "rewards/margins": 4.88486909866333, + "rewards/rejected": -4.789495944976807, + "step": 155, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.8994594594594595, + "grad_norm": 1.4425420761108398, + "learning_rate": 0.00016760468498337283, + "logits/chosen": -0.4141704738140106, + "logits/rejected": -0.753037691116333, + "logps/chosen": -2.9428489208221436, + "logps/rejected": -85.63114166259766, + "loss": 0.2691604495048523, + "memory(GiB)": 41.24, + "nll_loss": 0.22332294285297394, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.29303914308547974, + "rewards/margins": 6.752739906311035, + "rewards/rejected": -6.459700584411621, + "step": 156, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9052252252252252, + "grad_norm": 3.795686960220337, + "learning_rate": 0.00016713375981966612, + "logits/chosen": -0.4735071659088135, + "logits/rejected": -0.7943378686904907, + "logps/chosen": -9.046199798583984, + "logps/rejected": -86.67741394042969, + "loss": 0.602267324924469, + "memory(GiB)": 41.24, + "nll_loss": 0.5201234817504883, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.15314966440200806, + "rewards/margins": 6.5488057136535645, + "rewards/rejected": -6.7019548416137695, + "step": 157, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.910990990990991, + "grad_norm": 4.890588760375977, + "learning_rate": 0.00016666010853320543, + "logits/chosen": -0.5752784013748169, + "logits/rejected": -0.7697316408157349, + "logps/chosen": -9.034675598144531, + "logps/rejected": -79.48356628417969, + "loss": 0.7191926836967468, + "memory(GiB)": 41.24, + "nll_loss": 0.5265187621116638, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.07981552183628082, + "rewards/margins": 6.082505226135254, + "rewards/rejected": -6.162321090698242, + "step": 158, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9167567567567567, + "grad_norm": 1.312125563621521, + "learning_rate": 0.00016618375035770498, + "logits/chosen": -0.5153398513793945, + "logits/rejected": -0.8238294720649719, + "logps/chosen": -3.0522966384887695, + "logps/rejected": -84.81144714355469, + "loss": 0.32918140292167664, + "memory(GiB)": 41.24, + "nll_loss": 0.2058570384979248, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.19958336651325226, + "rewards/margins": 6.894275665283203, + "rewards/rejected": -6.694692611694336, + "step": 159, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9225225225225225, + "grad_norm": 1.3166769742965698, + "learning_rate": 0.00016570470463679856, + "logits/chosen": -0.4472997486591339, + "logits/rejected": -0.7798503041267395, + "logps/chosen": -2.2529990673065186, + "logps/rejected": -94.50987243652344, + "loss": 0.2068968117237091, + "memory(GiB)": 41.24, + "nll_loss": 0.13873136043548584, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15320263803005219, + "rewards/margins": 7.273505687713623, + "rewards/rejected": -7.120303153991699, + "step": 160, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9282882882882882, + "grad_norm": 1.4938724040985107, + "learning_rate": 0.00016522299082325382, + "logits/chosen": -0.4986165463924408, + "logits/rejected": -0.7801789045333862, + "logps/chosen": -2.9465794563293457, + "logps/rejected": -73.10553741455078, + "loss": 0.30796459317207336, + "memory(GiB)": 41.24, + "nll_loss": 0.2202310413122177, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.38857752084732056, + "rewards/margins": 5.931979656219482, + "rewards/rejected": -5.543401718139648, + "step": 161, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9340540540540541, + "grad_norm": 1.7879637479782104, + "learning_rate": 0.00016473862847818277, + "logits/chosen": -0.5118488073348999, + "logits/rejected": -0.8153480291366577, + "logps/chosen": -4.100103855133057, + "logps/rejected": -91.44921875, + "loss": 0.35479840636253357, + "memory(GiB)": 41.24, + "nll_loss": 0.30052870512008667, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14176566898822784, + "rewards/margins": 7.297792911529541, + "rewards/rejected": -7.156026840209961, + "step": 162, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9398198198198198, + "grad_norm": 2.240473985671997, + "learning_rate": 0.00016425163727024707, + "logits/chosen": -0.578972339630127, + "logits/rejected": -0.7973175048828125, + "logps/chosen": -4.10087776184082, + "logps/rejected": -75.45283508300781, + "loss": 0.41070497035980225, + "memory(GiB)": 41.24, + "nll_loss": 0.24780017137527466, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.003609701991081238, + "rewards/margins": 6.174129009246826, + "rewards/rejected": -6.17051887512207, + "step": 163, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9455855855855856, + "grad_norm": 1.820946455001831, + "learning_rate": 0.0001637620369748595, + "logits/chosen": -0.5063561201095581, + "logits/rejected": -0.839270293712616, + "logps/chosen": -3.7182536125183105, + "logps/rejected": -102.9184341430664, + "loss": 0.2363234907388687, + "memory(GiB)": 41.24, + "nll_loss": 0.16097372770309448, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07240395247936249, + "rewards/margins": 8.28852653503418, + "rewards/rejected": -8.216121673583984, + "step": 164, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9513513513513514, + "grad_norm": 1.6907538175582886, + "learning_rate": 0.00016326984747338095, + "logits/chosen": -0.5052527785301208, + "logits/rejected": -0.7384231686592102, + "logps/chosen": -4.822022914886475, + "logps/rejected": -71.9358901977539, + "loss": 0.4056674838066101, + "memory(GiB)": 41.24, + "nll_loss": 0.30248117446899414, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.24947428703308105, + "rewards/margins": 5.717057704925537, + "rewards/rejected": -5.467583179473877, + "step": 165, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9571171171171171, + "grad_norm": 2.038442850112915, + "learning_rate": 0.00016277508875231302, + "logits/chosen": -0.5684705972671509, + "logits/rejected": -0.7857838273048401, + "logps/chosen": -5.808892250061035, + "logps/rejected": -83.46935272216797, + "loss": 0.41192197799682617, + "memory(GiB)": 41.24, + "nll_loss": 0.3203110098838806, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.05355273187160492, + "rewards/margins": 6.413900375366211, + "rewards/rejected": -6.360347747802734, + "step": 166, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9628828828828829, + "grad_norm": 1.4706615209579468, + "learning_rate": 0.00016227778090248648, + "logits/chosen": -0.48619288206100464, + "logits/rejected": -0.7674732208251953, + "logps/chosen": -1.154846429824829, + "logps/rejected": -72.29942321777344, + "loss": 0.18448084592819214, + "memory(GiB)": 41.24, + "nll_loss": 0.1356724500656128, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2431672066450119, + "rewards/margins": 5.5912065505981445, + "rewards/rejected": -5.348039150238037, + "step": 167, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9686486486486486, + "grad_norm": 1.3399150371551514, + "learning_rate": 0.00016177794411824544, + "logits/chosen": -0.5658018589019775, + "logits/rejected": -0.8080291748046875, + "logps/chosen": -3.5855631828308105, + "logps/rejected": -70.36351013183594, + "loss": 0.33944055438041687, + "memory(GiB)": 41.24, + "nll_loss": 0.2699050009250641, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18274028599262238, + "rewards/margins": 5.833596229553223, + "rewards/rejected": -5.650856018066406, + "step": 168, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9744144144144145, + "grad_norm": 3.7518484592437744, + "learning_rate": 0.00016127559869662722, + "logits/chosen": -0.46642014384269714, + "logits/rejected": -0.8066117167472839, + "logps/chosen": -3.712211847305298, + "logps/rejected": -80.26597595214844, + "loss": 0.25077763199806213, + "memory(GiB)": 41.24, + "nll_loss": 0.19776861369609833, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.023500367999076843, + "rewards/margins": 6.045979022979736, + "rewards/rejected": -6.069479942321777, + "step": 169, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9801801801801802, + "grad_norm": 1.6079647541046143, + "learning_rate": 0.00016077076503653826, + "logits/chosen": -0.47919565439224243, + "logits/rejected": -0.7426984310150146, + "logps/chosen": -3.771472454071045, + "logps/rejected": -72.3167495727539, + "loss": 0.34788501262664795, + "memory(GiB)": 41.24, + "nll_loss": 0.26689645648002625, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13779021799564362, + "rewards/margins": 5.563295364379883, + "rewards/rejected": -5.4255051612854, + "step": 170, + "train_speed(iter/s)": 0.010714 + }, + { + "epoch": 0.985945945945946, + "grad_norm": 3.0278408527374268, + "learning_rate": 0.00016026346363792567, + "logits/chosen": -0.431247353553772, + "logits/rejected": -0.7434531450271606, + "logps/chosen": -3.02232027053833, + "logps/rejected": -71.19493103027344, + "loss": 0.3027627468109131, + "memory(GiB)": 41.24, + "nll_loss": 0.2018585056066513, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1917632669210434, + "rewards/margins": 5.246973991394043, + "rewards/rejected": -5.055210113525391, + "step": 171, + "train_speed(iter/s)": 0.010713 + }, + { + "epoch": 0.9917117117117117, + "grad_norm": 1.8670094013214111, + "learning_rate": 0.00015975371510094485, + "logits/chosen": -0.5508197546005249, + "logits/rejected": -0.7615008354187012, + "logps/chosen": -6.864560604095459, + "logps/rejected": -65.1205825805664, + "loss": 0.4220251142978668, + "memory(GiB)": 41.24, + "nll_loss": 0.38794711232185364, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.46989697217941284, + "rewards/margins": 5.622817039489746, + "rewards/rejected": -5.152919769287109, + "step": 172, + "train_speed(iter/s)": 0.010714 + }, + { + "epoch": 0.9974774774774775, + "grad_norm": 1.1435422897338867, + "learning_rate": 0.00015924154012512308, + "logits/chosen": -0.5378420948982239, + "logits/rejected": -0.837982177734375, + "logps/chosen": -2.298398971557617, + "logps/rejected": -76.451904296875, + "loss": 0.18354284763336182, + "memory(GiB)": 41.24, + "nll_loss": 0.1457872986793518, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08272568881511688, + "rewards/margins": 5.861605167388916, + "rewards/rejected": -5.778879165649414, + "step": 173, + "train_speed(iter/s)": 0.010714 + }, + { + "epoch": 1.0, + "grad_norm": 1.1435422897338867, + "learning_rate": 0.0001587269595085186, + "logits/chosen": -0.46139007806777954, + "logits/rejected": -0.787911593914032, + "logps/chosen": -2.2199597358703613, + "logps/rejected": -95.19585418701172, + "loss": 0.09489377588033676, + "memory(GiB)": 41.24, + "nll_loss": 0.14853443205356598, + "rewards/accuracies": 0.9285714030265808, + "rewards/chosen": 0.0750870555639267, + "rewards/margins": 7.632567882537842, + "rewards/rejected": -7.557480812072754, + "step": 174, + "train_speed(iter/s)": 0.010751 + }, + { + "epoch": 1.0057657657657657, + "grad_norm": 1.8974206447601318, + "learning_rate": 0.00015820999414687656, + "logits/chosen": -0.5884040594100952, + "logits/rejected": -0.821685254573822, + "logps/chosen": -2.4951252937316895, + "logps/rejected": -70.88331604003906, + "loss": 0.2344619631767273, + "memory(GiB)": 41.24, + "nll_loss": 0.19315816462039948, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15108144283294678, + "rewards/margins": 5.873671531677246, + "rewards/rejected": -5.72258996963501, + "step": 175, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0115315315315316, + "grad_norm": 0.6439478397369385, + "learning_rate": 0.00015769066503277997, + "logits/chosen": -0.52037113904953, + "logits/rejected": -0.7941011786460876, + "logps/chosen": -2.5466148853302, + "logps/rejected": -88.6285171508789, + "loss": 0.16956239938735962, + "memory(GiB)": 41.24, + "nll_loss": 0.14753001928329468, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19697943329811096, + "rewards/margins": 6.911043167114258, + "rewards/rejected": -6.714064121246338, + "step": 176, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0172972972972973, + "grad_norm": 0.8241242170333862, + "learning_rate": 0.0001571689932547978, + "logits/chosen": -0.5486235618591309, + "logits/rejected": -0.8218496441841125, + "logps/chosen": -4.344364643096924, + "logps/rejected": -78.90025329589844, + "loss": 0.23550763726234436, + "memory(GiB)": 41.24, + "nll_loss": 0.1828317493200302, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23757538199424744, + "rewards/margins": 6.283997058868408, + "rewards/rejected": -6.046421527862549, + "step": 177, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.023063063063063, + "grad_norm": 0.8632805943489075, + "learning_rate": 0.00015664499999662815, + "logits/chosen": -0.5551503896713257, + "logits/rejected": -0.8251141309738159, + "logps/chosen": -2.980712890625, + "logps/rejected": -83.22628784179688, + "loss": 0.15064574778079987, + "memory(GiB)": 41.24, + "nll_loss": 0.13642281293869019, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2624984383583069, + "rewards/margins": 6.998701572418213, + "rewards/rejected": -6.736203193664551, + "step": 178, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0288288288288288, + "grad_norm": 1.1632269620895386, + "learning_rate": 0.00015611870653623825, + "logits/chosen": -0.4921662211418152, + "logits/rejected": -0.8483188152313232, + "logps/chosen": -1.3713245391845703, + "logps/rejected": -85.42861938476562, + "loss": 0.1704276204109192, + "memory(GiB)": 41.24, + "nll_loss": 0.1270376294851303, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12698447704315186, + "rewards/margins": 6.958097457885742, + "rewards/rejected": -6.831112861633301, + "step": 179, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0345945945945947, + "grad_norm": 1.6630468368530273, + "learning_rate": 0.00015559013424500047, + "logits/chosen": -0.6110640168190002, + "logits/rejected": -0.8926798105239868, + "logps/chosen": -2.6163129806518555, + "logps/rejected": -90.13837432861328, + "loss": 0.14455236494541168, + "memory(GiB)": 41.24, + "nll_loss": 0.1374558061361313, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3422771096229553, + "rewards/margins": 7.565336227416992, + "rewards/rejected": -7.223059177398682, + "step": 180, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0403603603603604, + "grad_norm": 1.077152132987976, + "learning_rate": 0.0001550593045868244, + "logits/chosen": -0.603461742401123, + "logits/rejected": -0.8620364665985107, + "logps/chosen": -4.633229732513428, + "logps/rejected": -76.99014282226562, + "loss": 0.40596532821655273, + "memory(GiB)": 41.24, + "nll_loss": 0.3524565100669861, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.26899972558021545, + "rewards/margins": 6.21177864074707, + "rewards/rejected": -5.942779064178467, + "step": 181, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.046126126126126, + "grad_norm": 12.267168045043945, + "learning_rate": 0.00015452623911728523, + "logits/chosen": -0.6155453324317932, + "logits/rejected": -0.906221866607666, + "logps/chosen": -4.583263397216797, + "logps/rejected": -87.18157196044922, + "loss": 0.3373698592185974, + "memory(GiB)": 41.24, + "nll_loss": 0.3133515417575836, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26982182264328003, + "rewards/margins": 7.339932441711426, + "rewards/rejected": -7.070110321044922, + "step": 182, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 1.0518918918918918, + "grad_norm": 0.7905959486961365, + "learning_rate": 0.00015399095948274852, + "logits/chosen": -0.5579067468643188, + "logits/rejected": -0.8554020524024963, + "logps/chosen": -1.4490176439285278, + "logps/rejected": -91.41957092285156, + "loss": 0.12704932689666748, + "memory(GiB)": 41.24, + "nll_loss": 0.09653271734714508, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26760008931159973, + "rewards/margins": 7.848196029663086, + "rewards/rejected": -7.58059549331665, + "step": 183, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0576576576576577, + "grad_norm": 1.1468485593795776, + "learning_rate": 0.00015345348741949117, + "logits/chosen": -0.5733897089958191, + "logits/rejected": -0.9108660221099854, + "logps/chosen": -3.66715669631958, + "logps/rejected": -95.78589630126953, + "loss": 0.24044549465179443, + "memory(GiB)": 41.24, + "nll_loss": 0.21640615165233612, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16394057869911194, + "rewards/margins": 7.828693389892578, + "rewards/rejected": -7.664752960205078, + "step": 184, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0634234234234234, + "grad_norm": 2.031723976135254, + "learning_rate": 0.00015291384475281877, + "logits/chosen": -0.49676764011383057, + "logits/rejected": -0.8489770293235779, + "logps/chosen": -1.7427412271499634, + "logps/rejected": -96.12220764160156, + "loss": 0.12766067683696747, + "memory(GiB)": 41.24, + "nll_loss": 0.11571483314037323, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14926967024803162, + "rewards/margins": 7.909836769104004, + "rewards/rejected": -7.760568141937256, + "step": 185, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0691891891891891, + "grad_norm": 2.5470285415649414, + "learning_rate": 0.00015237205339617917, + "logits/chosen": -0.5048971772193909, + "logits/rejected": -0.7917367219924927, + "logps/chosen": -4.352810382843018, + "logps/rejected": -81.40713500976562, + "loss": 0.31023305654525757, + "memory(GiB)": 41.24, + "nll_loss": 0.2897438108921051, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14250265061855316, + "rewards/margins": 6.402338981628418, + "rewards/rejected": -6.259836196899414, + "step": 186, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0749549549549549, + "grad_norm": 1.091139316558838, + "learning_rate": 0.000151828135350273, + "logits/chosen": -0.5620720982551575, + "logits/rejected": -0.841740608215332, + "logps/chosen": -1.8221709728240967, + "logps/rejected": -82.46046447753906, + "loss": 0.15480715036392212, + "memory(GiB)": 41.24, + "nll_loss": 0.12393927574157715, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3422683775424957, + "rewards/margins": 6.792727470397949, + "rewards/rejected": -6.450459003448486, + "step": 187, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0807207207207208, + "grad_norm": 0.9704499244689941, + "learning_rate": 0.00015128211270215992, + "logits/chosen": -0.6280016899108887, + "logits/rejected": -0.8943365812301636, + "logps/chosen": -2.3877384662628174, + "logps/rejected": -81.13067626953125, + "loss": 0.18478192389011383, + "memory(GiB)": 41.24, + "nll_loss": 0.16890017688274384, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34920331835746765, + "rewards/margins": 6.807940483093262, + "rewards/rejected": -6.458736896514893, + "step": 188, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0864864864864865, + "grad_norm": 1.0640044212341309, + "learning_rate": 0.00015073400762436197, + "logits/chosen": -0.6785641312599182, + "logits/rejected": -0.8755528330802917, + "logps/chosen": -3.03802490234375, + "logps/rejected": -80.31719970703125, + "loss": 0.23102031648159027, + "memory(GiB)": 41.24, + "nll_loss": 0.20030677318572998, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.299454927444458, + "rewards/margins": 6.583179473876953, + "rewards/rejected": -6.283725261688232, + "step": 189, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.0922522522522522, + "grad_norm": 1.2426904439926147, + "learning_rate": 0.00015018384237396292, + "logits/chosen": -0.6075233221054077, + "logits/rejected": -0.9030817747116089, + "logps/chosen": -3.1166131496429443, + "logps/rejected": -88.71156311035156, + "loss": 0.1606086939573288, + "memory(GiB)": 41.24, + "nll_loss": 0.1422414779663086, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22253718972206116, + "rewards/margins": 7.480426788330078, + "rewards/rejected": -7.257890224456787, + "step": 190, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.098018018018018, + "grad_norm": 13.646331787109375, + "learning_rate": 0.0001496316392917049, + "logits/chosen": -0.5769588351249695, + "logits/rejected": -0.9033970236778259, + "logps/chosen": -1.5371689796447754, + "logps/rejected": -104.17735290527344, + "loss": 0.10281458497047424, + "memory(GiB)": 41.24, + "nll_loss": 0.09561118483543396, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2810855209827423, + "rewards/margins": 8.708104133605957, + "rewards/rejected": -8.427018165588379, + "step": 191, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.1037837837837838, + "grad_norm": 1.8991550207138062, + "learning_rate": 0.00014907742080108073, + "logits/chosen": -0.6996522545814514, + "logits/rejected": -0.9485704302787781, + "logps/chosen": -3.0516557693481445, + "logps/rejected": -86.58012390136719, + "loss": 0.4684174954891205, + "memory(GiB)": 41.24, + "nll_loss": 0.4487941563129425, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19735433161258698, + "rewards/margins": 7.334392547607422, + "rewards/rejected": -7.137038230895996, + "step": 192, + "train_speed(iter/s)": 0.01075 + }, + { + "epoch": 1.1095495495495495, + "grad_norm": 19.20090675354004, + "learning_rate": 0.0001485212094074237, + "logits/chosen": -0.576482892036438, + "logits/rejected": -0.84342360496521, + "logps/chosen": -4.030856132507324, + "logps/rejected": -86.30321502685547, + "loss": 0.6711006760597229, + "memory(GiB)": 41.24, + "nll_loss": 0.6366056799888611, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26511046290397644, + "rewards/margins": 7.336245536804199, + "rewards/rejected": -7.071135520935059, + "step": 193, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.1153153153153152, + "grad_norm": 0.9619101881980896, + "learning_rate": 0.0001479630276969936, + "logits/chosen": -0.6200534701347351, + "logits/rejected": -0.8699415922164917, + "logps/chosen": -3.693956136703491, + "logps/rejected": -93.85980987548828, + "loss": 0.09817313402891159, + "memory(GiB)": 41.24, + "nll_loss": 0.08495140075683594, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26861289143562317, + "rewards/margins": 8.128625869750977, + "rewards/rejected": -7.860013008117676, + "step": 194, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.1210810810810812, + "grad_norm": 0.7348951101303101, + "learning_rate": 0.00014740289833605939, + "logits/chosen": -0.6436381936073303, + "logits/rejected": -0.9239126443862915, + "logps/chosen": -3.7549028396606445, + "logps/rejected": -93.33834838867188, + "loss": 0.26953551173210144, + "memory(GiB)": 41.24, + "nll_loss": 0.24437427520751953, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1677757203578949, + "rewards/margins": 7.494346618652344, + "rewards/rejected": -7.326571464538574, + "step": 195, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.1268468468468469, + "grad_norm": 5.2597246170043945, + "learning_rate": 0.00014684084406997903, + "logits/chosen": -0.6959158778190613, + "logits/rejected": -0.9090582132339478, + "logps/chosen": -2.6071348190307617, + "logps/rejected": -81.92942810058594, + "loss": 0.1599394530057907, + "memory(GiB)": 41.24, + "nll_loss": 0.13409104943275452, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21387647092342377, + "rewards/margins": 6.953554630279541, + "rewards/rejected": -6.739677906036377, + "step": 196, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.1326126126126126, + "grad_norm": 0.9787479639053345, + "learning_rate": 0.0001462768877222757, + "logits/chosen": -0.5895656943321228, + "logits/rejected": -0.9225438833236694, + "logps/chosen": -1.224118947982788, + "logps/rejected": -100.8663558959961, + "loss": 0.06828276813030243, + "memory(GiB)": 41.24, + "nll_loss": 0.06501688808202744, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2447878122329712, + "rewards/margins": 8.5127534866333, + "rewards/rejected": -8.267965316772461, + "step": 197, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.1383783783783783, + "grad_norm": 0.7639157176017761, + "learning_rate": 0.00014571105219371102, + "logits/chosen": -0.6577166318893433, + "logits/rejected": -0.9819097518920898, + "logps/chosen": -0.7252616286277771, + "logps/rejected": -81.32587432861328, + "loss": 0.1278780996799469, + "memory(GiB)": 41.24, + "nll_loss": 0.10438857972621918, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18735525012016296, + "rewards/margins": 6.4731879234313965, + "rewards/rejected": -6.28583288192749, + "step": 198, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.1441441441441442, + "grad_norm": 0.956177830696106, + "learning_rate": 0.00014514336046135518, + "logits/chosen": -0.7244867086410522, + "logits/rejected": -0.9211379289627075, + "logps/chosen": -3.929692268371582, + "logps/rejected": -73.1805419921875, + "loss": 0.2801938056945801, + "memory(GiB)": 41.24, + "nll_loss": 0.22287607192993164, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23973776400089264, + "rewards/margins": 6.041521072387695, + "rewards/rejected": -5.801782608032227, + "step": 199, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.14990990990991, + "grad_norm": 0.9859761595726013, + "learning_rate": 0.00014457383557765386, + "logits/chosen": -0.6032741069793701, + "logits/rejected": -0.8709255456924438, + "logps/chosen": -2.3330612182617188, + "logps/rejected": -78.04156494140625, + "loss": 0.21972186863422394, + "memory(GiB)": 41.24, + "nll_loss": 0.16077226400375366, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6756027936935425, + "rewards/margins": 6.16083288192749, + "rewards/rejected": -5.485230445861816, + "step": 200, + "train_speed(iter/s)": 0.010749 + }, + { + "epoch": 1.1556756756756756, + "grad_norm": 2.053891658782959, + "learning_rate": 0.00014400250066949196, + "logits/chosen": -0.7468363642692566, + "logits/rejected": -0.9693096876144409, + "logps/chosen": -2.202683210372925, + "logps/rejected": -83.35052490234375, + "loss": 0.1414460837841034, + "memory(GiB)": 41.24, + "nll_loss": 0.1286502480506897, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2432466298341751, + "rewards/margins": 7.013325214385986, + "rewards/rejected": -6.770079612731934, + "step": 201, + "train_speed(iter/s)": 0.010738 + }, + { + "epoch": 1.1614414414414413, + "grad_norm": 0.7447286248207092, + "learning_rate": 0.00014342937893725488, + "logits/chosen": -0.6638346910476685, + "logits/rejected": -0.9504916071891785, + "logps/chosen": -1.4076967239379883, + "logps/rejected": -91.8344955444336, + "loss": 0.11076842993497849, + "memory(GiB)": 41.24, + "nll_loss": 0.0927545577287674, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2952735722064972, + "rewards/margins": 7.439762592315674, + "rewards/rejected": -7.144488334655762, + "step": 202, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.1672072072072073, + "grad_norm": 1.7305192947387695, + "learning_rate": 0.00014285449365388598, + "logits/chosen": -0.6913129687309265, + "logits/rejected": -1.0035713911056519, + "logps/chosen": -2.595938205718994, + "logps/rejected": -93.46743774414062, + "loss": 0.1786770224571228, + "memory(GiB)": 41.24, + "nll_loss": 0.15392035245895386, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16032256186008453, + "rewards/margins": 7.787504196166992, + "rewards/rejected": -7.627182483673096, + "step": 203, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.172972972972973, + "grad_norm": 0.7990829944610596, + "learning_rate": 0.00014227786816394184, + "logits/chosen": -0.7262576818466187, + "logits/rejected": -1.0095360279083252, + "logps/chosen": -4.604730606079102, + "logps/rejected": -94.75875091552734, + "loss": 0.1926305592060089, + "memory(GiB)": 41.24, + "nll_loss": 0.18425899744033813, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.40172773599624634, + "rewards/margins": 7.936948776245117, + "rewards/rejected": -7.535220623016357, + "step": 204, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.1787387387387387, + "grad_norm": 0.7804811596870422, + "learning_rate": 0.00014169952588264417, + "logits/chosen": -0.7492111921310425, + "logits/rejected": -1.052622675895691, + "logps/chosen": -2.9088637828826904, + "logps/rejected": -99.19158935546875, + "loss": 0.13855330646038055, + "memory(GiB)": 41.24, + "nll_loss": 0.12824688851833344, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23516345024108887, + "rewards/margins": 8.24970531463623, + "rewards/rejected": -8.014541625976562, + "step": 205, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.1845045045045044, + "grad_norm": 1.3688875436782837, + "learning_rate": 0.0001411194902949289, + "logits/chosen": -0.7413324117660522, + "logits/rejected": -1.0183649063110352, + "logps/chosen": -3.5715129375457764, + "logps/rejected": -105.68714141845703, + "loss": 0.2724800109863281, + "memory(GiB)": 41.24, + "nll_loss": 0.25694820284843445, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.313816636800766, + "rewards/margins": 8.691929817199707, + "rewards/rejected": -8.37811279296875, + "step": 206, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.1902702702702703, + "grad_norm": 2.0642895698547363, + "learning_rate": 0.0001405377849544927, + "logits/chosen": -0.6753020286560059, + "logits/rejected": -1.0321030616760254, + "logps/chosen": -2.917701005935669, + "logps/rejected": -96.5132827758789, + "loss": 0.17677046358585358, + "memory(GiB)": 41.24, + "nll_loss": 0.15049855411052704, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27416104078292847, + "rewards/margins": 7.89042854309082, + "rewards/rejected": -7.616267681121826, + "step": 207, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.196036036036036, + "grad_norm": 1.8087902069091797, + "learning_rate": 0.00013995443348283645, + "logits/chosen": -0.6556892395019531, + "logits/rejected": -0.991775631904602, + "logps/chosen": -2.722226858139038, + "logps/rejected": -102.24663543701172, + "loss": 0.24504928290843964, + "memory(GiB)": 41.24, + "nll_loss": 0.20121188461780548, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2066883146762848, + "rewards/margins": 8.376320838928223, + "rewards/rejected": -8.169632911682129, + "step": 208, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2018018018018017, + "grad_norm": 1.0671130418777466, + "learning_rate": 0.00013936945956830602, + "logits/chosen": -0.7256425023078918, + "logits/rejected": -1.002614140510559, + "logps/chosen": -3.5971474647521973, + "logps/rejected": -90.5836181640625, + "loss": 0.22293436527252197, + "memory(GiB)": 41.24, + "nll_loss": 0.18081282079219818, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19337910413742065, + "rewards/margins": 7.3919477462768555, + "rewards/rejected": -7.198568344116211, + "step": 209, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2075675675675677, + "grad_norm": 1.3437083959579468, + "learning_rate": 0.00013878288696513022, + "logits/chosen": -0.7431738376617432, + "logits/rejected": -1.0425618886947632, + "logps/chosen": -2.174896240234375, + "logps/rejected": -93.76087951660156, + "loss": 0.15365180373191833, + "memory(GiB)": 41.24, + "nll_loss": 0.14746138453483582, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2575794756412506, + "rewards/margins": 7.875104904174805, + "rewards/rejected": -7.617526054382324, + "step": 210, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2133333333333334, + "grad_norm": 0.76457679271698, + "learning_rate": 0.00013819473949245654, + "logits/chosen": -0.6461102962493896, + "logits/rejected": -1.0543663501739502, + "logps/chosen": -0.8342275619506836, + "logps/rejected": -104.50704193115234, + "loss": 0.09112594276666641, + "memory(GiB)": 41.24, + "nll_loss": 0.08857303112745285, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23656663298606873, + "rewards/margins": 8.866386413574219, + "rewards/rejected": -8.629819869995117, + "step": 211, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.219099099099099, + "grad_norm": 0.6871019005775452, + "learning_rate": 0.0001376050410333836, + "logits/chosen": -0.6719679236412048, + "logits/rejected": -0.9832717776298523, + "logps/chosen": -3.1014583110809326, + "logps/rejected": -98.14430236816406, + "loss": 0.3030906915664673, + "memory(GiB)": 41.24, + "nll_loss": 0.21772870421409607, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.02887733280658722, + "rewards/margins": 7.6625895500183105, + "rewards/rejected": -7.633711814880371, + "step": 212, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2248648648648648, + "grad_norm": 2.0604000091552734, + "learning_rate": 0.00013701381553399145, + "logits/chosen": -0.6421749591827393, + "logits/rejected": -1.0211272239685059, + "logps/chosen": -2.438856840133667, + "logps/rejected": -86.31884002685547, + "loss": 0.20499145984649658, + "memory(GiB)": 41.24, + "nll_loss": 0.1656690537929535, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1947031468153, + "rewards/margins": 7.004463195800781, + "rewards/rejected": -6.809760093688965, + "step": 213, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2306306306306307, + "grad_norm": 2.321046829223633, + "learning_rate": 0.00013642108700236916, + "logits/chosen": -0.7420238852500916, + "logits/rejected": -1.0413094758987427, + "logps/chosen": -0.9865455031394958, + "logps/rejected": -72.17105102539062, + "loss": 0.13595286011695862, + "memory(GiB)": 41.24, + "nll_loss": 0.10351557284593582, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11004337668418884, + "rewards/margins": 5.79013204574585, + "rewards/rejected": -5.680088520050049, + "step": 214, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2363963963963964, + "grad_norm": 4.301558494567871, + "learning_rate": 0.00013582687950764, + "logits/chosen": -0.6356464624404907, + "logits/rejected": -0.986944854259491, + "logps/chosen": -1.5492937564849854, + "logps/rejected": -79.20093536376953, + "loss": 0.19731514155864716, + "memory(GiB)": 41.24, + "nll_loss": 0.15034356713294983, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15903238952159882, + "rewards/margins": 6.342228412628174, + "rewards/rejected": -6.1831955909729, + "step": 215, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2421621621621621, + "grad_norm": 1.1075836420059204, + "learning_rate": 0.00013523121717898387, + "logits/chosen": -0.6029863953590393, + "logits/rejected": -0.9658622741699219, + "logps/chosen": -3.6208183765411377, + "logps/rejected": -97.2186508178711, + "loss": 0.4900016784667969, + "memory(GiB)": 41.24, + "nll_loss": 0.44314447045326233, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.04553697258234024, + "rewards/margins": 7.420025825500488, + "rewards/rejected": -7.374488830566406, + "step": 216, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2479279279279278, + "grad_norm": 2.592005968093872, + "learning_rate": 0.00013463412420465767, + "logits/chosen": -0.6447511911392212, + "logits/rejected": -0.9609813094139099, + "logps/chosen": -2.2061100006103516, + "logps/rejected": -88.9801254272461, + "loss": 0.21441426873207092, + "memory(GiB)": 41.24, + "nll_loss": 0.20333540439605713, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2582980692386627, + "rewards/margins": 6.966536521911621, + "rewards/rejected": -6.70823860168457, + "step": 217, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2536936936936938, + "grad_norm": 1.5074249505996704, + "learning_rate": 0.00013403562483101298, + "logits/chosen": -0.66890949010849, + "logits/rejected": -0.8881216049194336, + "logps/chosen": -4.03311014175415, + "logps/rejected": -70.53189086914062, + "loss": 0.306526780128479, + "memory(GiB)": 41.24, + "nll_loss": 0.22115619480609894, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.29509076476097107, + "rewards/margins": 5.955531120300293, + "rewards/rejected": -5.660440444946289, + "step": 218, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2594594594594595, + "grad_norm": 0.588188886642456, + "learning_rate": 0.00013343574336151153, + "logits/chosen": -0.6300919055938721, + "logits/rejected": -0.9817159175872803, + "logps/chosen": -2.001368761062622, + "logps/rejected": -94.01361083984375, + "loss": 0.3623042702674866, + "memory(GiB)": 41.24, + "nll_loss": 0.2725469172000885, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15840183198451996, + "rewards/margins": 7.825695991516113, + "rewards/rejected": -7.667293548583984, + "step": 219, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2652252252252252, + "grad_norm": 14.042351722717285, + "learning_rate": 0.0001328345041557382, + "logits/chosen": -0.759201169013977, + "logits/rejected": -1.016777515411377, + "logps/chosen": -3.4606502056121826, + "logps/rejected": -93.0523910522461, + "loss": 0.20867016911506653, + "memory(GiB)": 41.24, + "nll_loss": 0.15421104431152344, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11695647984743118, + "rewards/margins": 7.954014301300049, + "rewards/rejected": -7.8370585441589355, + "step": 220, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2709909909909909, + "grad_norm": 1.2167177200317383, + "learning_rate": 0.000132231931628412, + "logits/chosen": -0.5667409896850586, + "logits/rejected": -0.8965315222740173, + "logps/chosen": -3.431696891784668, + "logps/rejected": -105.54657745361328, + "loss": 0.1707317978143692, + "memory(GiB)": 41.24, + "nll_loss": 0.1522923707962036, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24732309579849243, + "rewards/margins": 8.84961223602295, + "rewards/rejected": -8.602288246154785, + "step": 221, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2767567567567568, + "grad_norm": 0.9449612498283386, + "learning_rate": 0.00013162805024839448, + "logits/chosen": -0.6432092189788818, + "logits/rejected": -0.8541032075881958, + "logps/chosen": -3.5057826042175293, + "logps/rejected": -85.99465942382812, + "loss": 0.2346688210964203, + "memory(GiB)": 41.24, + "nll_loss": 0.19423571228981018, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5884712338447571, + "rewards/margins": 7.24459981918335, + "rewards/rejected": -6.656128883361816, + "step": 222, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.2825225225225225, + "grad_norm": 1.486234426498413, + "learning_rate": 0.00013102288453769632, + "logits/chosen": -0.6213188171386719, + "logits/rejected": -0.9732288718223572, + "logps/chosen": -2.9368274211883545, + "logps/rejected": -108.50143432617188, + "loss": 0.19141153991222382, + "memory(GiB)": 41.24, + "nll_loss": 0.1635173112154007, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09779652208089828, + "rewards/margins": 8.9135103225708, + "rewards/rejected": -8.815712928771973, + "step": 223, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.2882882882882882, + "grad_norm": 1.301498532295227, + "learning_rate": 0.0001304164590704813, + "logits/chosen": -0.6701276302337646, + "logits/rejected": -0.9731598496437073, + "logps/chosen": -3.8457467555999756, + "logps/rejected": -93.66019439697266, + "loss": 0.28035908937454224, + "memory(GiB)": 41.24, + "nll_loss": 0.24942338466644287, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.4500879943370819, + "rewards/margins": 8.23204517364502, + "rewards/rejected": -7.781956672668457, + "step": 224, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.2940540540540542, + "grad_norm": 1.0099408626556396, + "learning_rate": 0.0001298087984720687, + "logits/chosen": -0.7134125232696533, + "logits/rejected": -0.9936617612838745, + "logps/chosen": -2.324589967727661, + "logps/rejected": -90.27128601074219, + "loss": 0.39343133568763733, + "memory(GiB)": 41.24, + "nll_loss": 0.2453940212726593, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.25489863753318787, + "rewards/margins": 7.6775102615356445, + "rewards/rejected": -7.422612190246582, + "step": 225, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.2998198198198199, + "grad_norm": 2.37233829498291, + "learning_rate": 0.00012919992741793307, + "logits/chosen": -0.6083082556724548, + "logits/rejected": -0.9392067193984985, + "logps/chosen": -2.33506441116333, + "logps/rejected": -91.54042053222656, + "loss": 0.18458783626556396, + "memory(GiB)": 41.24, + "nll_loss": 0.17687150835990906, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32263070344924927, + "rewards/margins": 7.811326503753662, + "rewards/rejected": -7.488696098327637, + "step": 226, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3055855855855856, + "grad_norm": 0.9204962253570557, + "learning_rate": 0.0001285898706327023, + "logits/chosen": -0.6583359837532043, + "logits/rejected": -0.9856835007667542, + "logps/chosen": -2.480973720550537, + "logps/rejected": -98.34219360351562, + "loss": 0.22872069478034973, + "memory(GiB)": 41.24, + "nll_loss": 0.2118297517299652, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.37609827518463135, + "rewards/margins": 8.75016975402832, + "rewards/rejected": -8.37407112121582, + "step": 227, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3113513513513513, + "grad_norm": 1.0065797567367554, + "learning_rate": 0.0001279786528891538, + "logits/chosen": -0.564249575138092, + "logits/rejected": -0.8596559762954712, + "logps/chosen": -2.2762789726257324, + "logps/rejected": -94.75420379638672, + "loss": 0.09863093495368958, + "memory(GiB)": 41.24, + "nll_loss": 0.09180156141519547, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.30846285820007324, + "rewards/margins": 8.076614379882812, + "rewards/rejected": -7.76815128326416, + "step": 228, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.317117117117117, + "grad_norm": 15.840254783630371, + "learning_rate": 0.0001273662990072083, + "logits/chosen": -0.6420232057571411, + "logits/rejected": -0.9717917442321777, + "logps/chosen": -4.723894119262695, + "logps/rejected": -91.68743896484375, + "loss": 0.5773552656173706, + "memory(GiB)": 41.24, + "nll_loss": 0.4086422324180603, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09256955981254578, + "rewards/margins": 7.509539604187012, + "rewards/rejected": -7.416969299316406, + "step": 229, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.322882882882883, + "grad_norm": 0.9773613810539246, + "learning_rate": 0.00012675283385292212, + "logits/chosen": -0.5419533252716064, + "logits/rejected": -0.908346951007843, + "logps/chosen": -2.8323910236358643, + "logps/rejected": -92.77593994140625, + "loss": 0.18030104041099548, + "memory(GiB)": 41.24, + "nll_loss": 0.1703605055809021, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3452431559562683, + "rewards/margins": 7.826876163482666, + "rewards/rejected": -7.48163366317749, + "step": 230, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3286486486486486, + "grad_norm": 0.5861178636550903, + "learning_rate": 0.00012613828233747727, + "logits/chosen": -0.5227465629577637, + "logits/rejected": -0.867357611656189, + "logps/chosen": -2.319690704345703, + "logps/rejected": -90.09024810791016, + "loss": 0.17545996606349945, + "memory(GiB)": 41.24, + "nll_loss": 0.14297524094581604, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15114043653011322, + "rewards/margins": 7.286042213439941, + "rewards/rejected": -7.134902477264404, + "step": 231, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3344144144144146, + "grad_norm": 1.1118627786636353, + "learning_rate": 0.00012552266941617018, + "logits/chosen": -0.6113250851631165, + "logits/rejected": -0.7813447713851929, + "logps/chosen": -4.297638893127441, + "logps/rejected": -73.71047973632812, + "loss": 0.2730986773967743, + "memory(GiB)": 41.24, + "nll_loss": 0.2158653289079666, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24944612383842468, + "rewards/margins": 6.196266174316406, + "rewards/rejected": -5.946820259094238, + "step": 232, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3401801801801803, + "grad_norm": 0.8542975783348083, + "learning_rate": 0.000124906020087398, + "logits/chosen": -0.45566728711128235, + "logits/rejected": -0.8274158239364624, + "logps/chosen": -1.3975021839141846, + "logps/rejected": -92.9134750366211, + "loss": 0.10510716587305069, + "memory(GiB)": 41.24, + "nll_loss": 0.08654335141181946, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.292056143283844, + "rewards/margins": 7.4350056648254395, + "rewards/rejected": -7.142949104309082, + "step": 233, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.345945945945946, + "grad_norm": 0.6710683703422546, + "learning_rate": 0.00012428835939164363, + "logits/chosen": -0.48423632979393005, + "logits/rejected": -0.931830883026123, + "logps/chosen": -0.9655717611312866, + "logps/rejected": -103.34365844726562, + "loss": 0.10704699903726578, + "memory(GiB)": 41.24, + "nll_loss": 0.10177168250083923, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27281564474105835, + "rewards/margins": 8.59388256072998, + "rewards/rejected": -8.321066856384277, + "step": 234, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3517117117117117, + "grad_norm": 1.0295995473861694, + "learning_rate": 0.00012366971241045894, + "logits/chosen": -0.5070984959602356, + "logits/rejected": -0.8951232433319092, + "logps/chosen": -1.928631067276001, + "logps/rejected": -88.40235900878906, + "loss": 0.21813911199569702, + "memory(GiB)": 41.24, + "nll_loss": 0.16451862454414368, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.064000703394413, + "rewards/margins": 7.2806854248046875, + "rewards/rejected": -7.216684341430664, + "step": 235, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3574774774774774, + "grad_norm": 0.8053532242774963, + "learning_rate": 0.00012305010426544614, + "logits/chosen": -0.5998705625534058, + "logits/rejected": -0.9042835831642151, + "logps/chosen": -4.446467876434326, + "logps/rejected": -86.84009552001953, + "loss": 0.25722736120224, + "memory(GiB)": 41.24, + "nll_loss": 0.24677737057209015, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.43149375915527344, + "rewards/margins": 7.540010929107666, + "rewards/rejected": -7.108516693115234, + "step": 236, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3632432432432433, + "grad_norm": 1.9170420169830322, + "learning_rate": 0.00012242956011723782, + "logits/chosen": -0.5368235111236572, + "logits/rejected": -0.8908076286315918, + "logps/chosen": -1.4597830772399902, + "logps/rejected": -100.62641906738281, + "loss": 0.1386258900165558, + "memory(GiB)": 41.24, + "nll_loss": 0.13322040438652039, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2073771357536316, + "rewards/margins": 8.687051773071289, + "rewards/rejected": -8.479674339294434, + "step": 237, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.369009009009009, + "grad_norm": 1.0038492679595947, + "learning_rate": 0.00012180810516447512, + "logits/chosen": -0.5559844970703125, + "logits/rejected": -0.8453220725059509, + "logps/chosen": -3.5673227310180664, + "logps/rejected": -92.9088134765625, + "loss": 0.23567795753479004, + "memory(GiB)": 41.24, + "nll_loss": 0.20739604532718658, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27056020498275757, + "rewards/margins": 7.564149379730225, + "rewards/rejected": -7.293589115142822, + "step": 238, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.3747747747747747, + "grad_norm": 0.7483298182487488, + "learning_rate": 0.0001211857646427845, + "logits/chosen": -0.5241140723228455, + "logits/rejected": -0.8691558837890625, + "logps/chosen": -1.9950472116470337, + "logps/rejected": -82.87139892578125, + "loss": 0.15872085094451904, + "memory(GiB)": 41.24, + "nll_loss": 0.12550276517868042, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3112674951553345, + "rewards/margins": 6.984847545623779, + "rewards/rejected": -6.673579216003418, + "step": 239, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.3805405405405407, + "grad_norm": 0.9307670593261719, + "learning_rate": 0.00012056256382375308, + "logits/chosen": -0.5428832769393921, + "logits/rejected": -0.9613853096961975, + "logps/chosen": -1.8748772144317627, + "logps/rejected": -97.14224243164062, + "loss": 0.12880003452301025, + "memory(GiB)": 41.24, + "nll_loss": 0.12042552977800369, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31744644045829773, + "rewards/margins": 8.378560066223145, + "rewards/rejected": -8.061113357543945, + "step": 240, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.3863063063063064, + "grad_norm": 2.9521400928497314, + "learning_rate": 0.00011993852801390226, + "logits/chosen": -0.4635058045387268, + "logits/rejected": -0.8949097394943237, + "logps/chosen": -0.5122127532958984, + "logps/rejected": -96.43069458007812, + "loss": 0.10679192841053009, + "memory(GiB)": 41.24, + "nll_loss": 0.053764842450618744, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.165335014462471, + "rewards/margins": 8.006298065185547, + "rewards/rejected": -7.840963840484619, + "step": 241, + "train_speed(iter/s)": 0.010737 + }, + { + "epoch": 1.392072072072072, + "grad_norm": 1.0468616485595703, + "learning_rate": 0.00011931368255366027, + "logits/chosen": -0.45876914262771606, + "logits/rejected": -0.831932544708252, + "logps/chosen": -2.4560790061950684, + "logps/rejected": -97.22357177734375, + "loss": 0.2074245661497116, + "memory(GiB)": 41.24, + "nll_loss": 0.17626309394836426, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24911226332187653, + "rewards/margins": 8.137984275817871, + "rewards/rejected": -7.888872146606445, + "step": 242, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.3978378378378378, + "grad_norm": 0.7075209021568298, + "learning_rate": 0.00011868805281633304, + "logits/chosen": -0.58433997631073, + "logits/rejected": -0.9207597970962524, + "logps/chosen": -2.2052416801452637, + "logps/rejected": -101.76717376708984, + "loss": 0.18332861363887787, + "memory(GiB)": 41.24, + "nll_loss": 0.1732126772403717, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4298679530620575, + "rewards/margins": 8.984416961669922, + "rewards/rejected": -8.554548263549805, + "step": 243, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.4036036036036035, + "grad_norm": 0.9815582036972046, + "learning_rate": 0.00011806166420707392, + "logits/chosen": -0.48037832975387573, + "logits/rejected": -0.9194313883781433, + "logps/chosen": -1.8615732192993164, + "logps/rejected": -102.38887023925781, + "loss": 0.21181951463222504, + "memory(GiB)": 41.24, + "nll_loss": 0.15694458782672882, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.29181793332099915, + "rewards/margins": 8.388344764709473, + "rewards/rejected": -8.096527099609375, + "step": 244, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.4093693693693694, + "grad_norm": 2.933603525161743, + "learning_rate": 0.00011743454216185201, + "logits/chosen": -0.640617311000824, + "logits/rejected": -0.9676884412765503, + "logps/chosen": -3.7124361991882324, + "logps/rejected": -97.49540710449219, + "loss": 0.2786301076412201, + "memory(GiB)": 41.24, + "nll_loss": 0.26465094089508057, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17746347188949585, + "rewards/margins": 8.408309936523438, + "rewards/rejected": -8.23084545135498, + "step": 245, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.4151351351351351, + "grad_norm": 0.6771543025970459, + "learning_rate": 0.00011680671214641927, + "logits/chosen": -0.4917660057544708, + "logits/rejected": -0.9493937492370605, + "logps/chosen": -1.4353026151657104, + "logps/rejected": -133.36654663085938, + "loss": 0.10904774069786072, + "memory(GiB)": 41.24, + "nll_loss": 0.10567378252744675, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24828633666038513, + "rewards/margins": 11.476335525512695, + "rewards/rejected": -11.228048324584961, + "step": 246, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.4209009009009008, + "grad_norm": 0.5642514824867249, + "learning_rate": 0.0001161781996552765, + "logits/chosen": -0.5594913363456726, + "logits/rejected": -1.0035367012023926, + "logps/chosen": -2.4414238929748535, + "logps/rejected": -129.03379821777344, + "loss": 0.1480759233236313, + "memory(GiB)": 41.24, + "nll_loss": 0.1389404833316803, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.01771201565861702, + "rewards/margins": 11.155719757080078, + "rewards/rejected": -11.138009071350098, + "step": 247, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.4266666666666667, + "grad_norm": 2.8613696098327637, + "learning_rate": 0.0001155490302106379, + "logits/chosen": -0.5504012703895569, + "logits/rejected": -0.993097722530365, + "logps/chosen": -1.1884583234786987, + "logps/rejected": -113.57362365722656, + "loss": 0.11941150575876236, + "memory(GiB)": 41.24, + "nll_loss": 0.10435314476490021, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20970577001571655, + "rewards/margins": 9.547948837280273, + "rewards/rejected": -9.338242530822754, + "step": 248, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.4324324324324325, + "grad_norm": 0.8018273711204529, + "learning_rate": 0.00011491922936139499, + "logits/chosen": -0.5908212661743164, + "logits/rejected": -0.9416083693504333, + "logps/chosen": -2.546377182006836, + "logps/rejected": -98.2177963256836, + "loss": 0.20611125230789185, + "memory(GiB)": 41.24, + "nll_loss": 0.19133996963500977, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17961813509464264, + "rewards/margins": 8.596184730529785, + "rewards/rejected": -8.416565895080566, + "step": 249, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.4381981981981982, + "grad_norm": 1.2365566492080688, + "learning_rate": 0.00011428882268207872, + "logits/chosen": -0.6211739778518677, + "logits/rejected": -0.8576729893684387, + "logps/chosen": -6.602561950683594, + "logps/rejected": -92.08356475830078, + "loss": 0.3254232406616211, + "memory(GiB)": 41.24, + "nll_loss": 0.2950092554092407, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.45778965950012207, + "rewards/margins": 8.364259719848633, + "rewards/rejected": -7.90647029876709, + "step": 250, + "train_speed(iter/s)": 0.010736 + }, + { + "epoch": 1.4439639639639639, + "grad_norm": 0.8326837420463562, + "learning_rate": 0.00011365783577182132, + "logits/chosen": -0.5177187919616699, + "logits/rejected": -0.9024335145950317, + "logps/chosen": -2.21883225440979, + "logps/rejected": -105.33366394042969, + "loss": 0.1589798927307129, + "memory(GiB)": 41.24, + "nll_loss": 0.13007135689258575, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3507290780544281, + "rewards/margins": 9.073866844177246, + "rewards/rejected": -8.723137855529785, + "step": 251, + "train_speed(iter/s)": 0.010728 + }, + { + "epoch": 1.4497297297297298, + "grad_norm": 1.8344132900238037, + "learning_rate": 0.00011302629425331666, + "logits/chosen": -0.603495180606842, + "logits/rejected": -0.801387369632721, + "logps/chosen": -5.291406154632568, + "logps/rejected": -84.78588104248047, + "loss": 0.3241059482097626, + "memory(GiB)": 41.24, + "nll_loss": 0.2689138948917389, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2446574866771698, + "rewards/margins": 7.256671905517578, + "rewards/rejected": -7.012013912200928, + "step": 252, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.4554954954954955, + "grad_norm": 0.6599681377410889, + "learning_rate": 0.00011239422377177973, + "logits/chosen": -0.5089681148529053, + "logits/rejected": -0.8658185005187988, + "logps/chosen": -3.079047679901123, + "logps/rejected": -127.54818725585938, + "loss": 0.17893767356872559, + "memory(GiB)": 41.24, + "nll_loss": 0.17471365630626678, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5883893370628357, + "rewards/margins": 11.444487571716309, + "rewards/rejected": -10.856100082397461, + "step": 253, + "train_speed(iter/s)": 0.010728 + }, + { + "epoch": 1.4612612612612612, + "grad_norm": 0.6386237144470215, + "learning_rate": 0.00011176164999390522, + "logits/chosen": -0.5488162040710449, + "logits/rejected": -0.9546905755996704, + "logps/chosen": -3.4519522190093994, + "logps/rejected": -109.98165893554688, + "loss": 0.22669866681098938, + "memory(GiB)": 41.24, + "nll_loss": 0.20005784928798676, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3646531105041504, + "rewards/margins": 9.544320106506348, + "rewards/rejected": -9.179666519165039, + "step": 254, + "train_speed(iter/s)": 0.010728 + }, + { + "epoch": 1.4670270270270271, + "grad_norm": 17.654590606689453, + "learning_rate": 0.00011112859860682547, + "logits/chosen": -0.5480679869651794, + "logits/rejected": -0.9488638639450073, + "logps/chosen": -3.5623536109924316, + "logps/rejected": -115.0560073852539, + "loss": 0.4454716145992279, + "memory(GiB)": 41.24, + "nll_loss": 0.3780556321144104, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10135876387357712, + "rewards/margins": 9.828811645507812, + "rewards/rejected": -9.727452278137207, + "step": 255, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.4727927927927928, + "grad_norm": 0.6790575385093689, + "learning_rate": 0.0001104950953170672, + "logits/chosen": -0.5322855710983276, + "logits/rejected": -0.8911909461021423, + "logps/chosen": -2.056039571762085, + "logps/rejected": -107.23054504394531, + "loss": 0.2234971523284912, + "memory(GiB)": 41.24, + "nll_loss": 0.20333045721054077, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2641701102256775, + "rewards/margins": 9.115086555480957, + "rewards/rejected": -8.850915908813477, + "step": 256, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.4785585585585586, + "grad_norm": 0.8362141847610474, + "learning_rate": 0.00010986116584950774, + "logits/chosen": -0.46502387523651123, + "logits/rejected": -0.8917255401611328, + "logps/chosen": -1.898627758026123, + "logps/rejected": -115.86322021484375, + "loss": 0.12826178967952728, + "memory(GiB)": 41.24, + "nll_loss": 0.11593392491340637, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.29406896233558655, + "rewards/margins": 9.646780014038086, + "rewards/rejected": -9.352710723876953, + "step": 257, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.4843243243243243, + "grad_norm": 0.681954026222229, + "learning_rate": 0.00010922683594633021, + "logits/chosen": -0.5689491033554077, + "logits/rejected": -0.8625025153160095, + "logps/chosen": -1.7617994546890259, + "logps/rejected": -99.32975769042969, + "loss": 0.1266973614692688, + "memory(GiB)": 41.24, + "nll_loss": 0.11154472082853317, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.283980131149292, + "rewards/margins": 8.681015014648438, + "rewards/rejected": -8.397034645080566, + "step": 258, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.49009009009009, + "grad_norm": 0.6331071853637695, + "learning_rate": 0.00010859213136597853, + "logits/chosen": -0.5191855430603027, + "logits/rejected": -0.8886591196060181, + "logps/chosen": -1.3958189487457275, + "logps/rejected": -97.13671112060547, + "loss": 0.13129664957523346, + "memory(GiB)": 41.24, + "nll_loss": 0.10424058139324188, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.33293408155441284, + "rewards/margins": 8.300026893615723, + "rewards/rejected": -7.967093467712402, + "step": 259, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.495855855855856, + "grad_norm": 0.7260443568229675, + "learning_rate": 0.00010795707788211118, + "logits/chosen": -0.4627129137516022, + "logits/rejected": -0.824970543384552, + "logps/chosen": -3.0662460327148438, + "logps/rejected": -102.18391418457031, + "loss": 0.20565594732761383, + "memory(GiB)": 41.24, + "nll_loss": 0.18343128263950348, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.37244486808776855, + "rewards/margins": 8.65424919128418, + "rewards/rejected": -8.281805038452148, + "step": 260, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.5016216216216216, + "grad_norm": 0.8002501726150513, + "learning_rate": 0.00010732170128255468, + "logits/chosen": -0.4538012444972992, + "logits/rejected": -0.7942509651184082, + "logps/chosen": -3.3011088371276855, + "logps/rejected": -107.73184204101562, + "loss": 0.23947054147720337, + "memory(GiB)": 41.24, + "nll_loss": 0.18957434594631195, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.30125465989112854, + "rewards/margins": 8.832905769348145, + "rewards/rejected": -8.531651496887207, + "step": 261, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.5073873873873875, + "grad_norm": 0.789520800113678, + "learning_rate": 0.00010668602736825641, + "logits/chosen": -0.5109374523162842, + "logits/rejected": -0.8500669002532959, + "logps/chosen": -1.8806393146514893, + "logps/rejected": -104.79667663574219, + "loss": 0.20148895680904388, + "memory(GiB)": 41.24, + "nll_loss": 0.14439257979393005, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4746500253677368, + "rewards/margins": 9.070365905761719, + "rewards/rejected": -8.595715522766113, + "step": 262, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.5131531531531532, + "grad_norm": 0.6022177338600159, + "learning_rate": 0.00010605008195223694, + "logits/chosen": -0.5372226238250732, + "logits/rejected": -0.8244591951370239, + "logps/chosen": -1.747965693473816, + "logps/rejected": -95.3003158569336, + "loss": 0.14818598330020905, + "memory(GiB)": 41.24, + "nll_loss": 0.14205510914325714, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.252836138010025, + "rewards/margins": 8.310218811035156, + "rewards/rejected": -8.057382583618164, + "step": 263, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.518918918918919, + "grad_norm": 0.893842339515686, + "learning_rate": 0.00010541389085854176, + "logits/chosen": -0.5556601285934448, + "logits/rejected": -0.8188039660453796, + "logps/chosen": -3.6805174350738525, + "logps/rejected": -88.07212829589844, + "loss": 0.3132077157497406, + "memory(GiB)": 41.24, + "nll_loss": 0.28640279173851013, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16909292340278625, + "rewards/margins": 7.427695274353027, + "rewards/rejected": -7.258602619171143, + "step": 264, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.5246846846846847, + "grad_norm": 2.960935592651367, + "learning_rate": 0.00010477747992119273, + "logits/chosen": -0.47233614325523376, + "logits/rejected": -0.8489798903465271, + "logps/chosen": -2.1952314376831055, + "logps/rejected": -109.46432495117188, + "loss": 0.1649601310491562, + "memory(GiB)": 41.24, + "nll_loss": 0.15151429176330566, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3918912410736084, + "rewards/margins": 9.63163948059082, + "rewards/rejected": -9.23974895477295, + "step": 265, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.5304504504504504, + "grad_norm": 1.122962474822998, + "learning_rate": 0.0001041408749831389, + "logits/chosen": -0.3698769807815552, + "logits/rejected": -0.8481926321983337, + "logps/chosen": -3.5086236000061035, + "logps/rejected": -113.31342315673828, + "loss": 0.5645697116851807, + "memory(GiB)": 41.24, + "nll_loss": 0.42728477716445923, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08735576272010803, + "rewards/margins": 9.291422843933105, + "rewards/rejected": -9.20406723022461, + "step": 266, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.536216216216216, + "grad_norm": 8.757622718811035, + "learning_rate": 0.00010350410189520723, + "logits/chosen": -0.48632851243019104, + "logits/rejected": -0.8631247282028198, + "logps/chosen": -2.381420850753784, + "logps/rejected": -115.440673828125, + "loss": 0.17880821228027344, + "memory(GiB)": 41.24, + "nll_loss": 0.1437128782272339, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3020068407058716, + "rewards/margins": 9.818366050720215, + "rewards/rejected": -9.516358375549316, + "step": 267, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.541981981981982, + "grad_norm": 1.859763264656067, + "learning_rate": 0.00010286718651505275, + "logits/chosen": -0.6271436214447021, + "logits/rejected": -0.9017850756645203, + "logps/chosen": -3.4276139736175537, + "logps/rejected": -94.77530670166016, + "loss": 0.23817601799964905, + "memory(GiB)": 41.24, + "nll_loss": 0.23029769957065582, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2481326013803482, + "rewards/margins": 8.214079856872559, + "rewards/rejected": -7.965947151184082, + "step": 268, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.5477477477477477, + "grad_norm": 3.0863282680511475, + "learning_rate": 0.00010223015470610871, + "logits/chosen": -0.5537242889404297, + "logits/rejected": -0.8814469575881958, + "logps/chosen": -4.503087520599365, + "logps/rejected": -90.9027328491211, + "loss": 0.3578626811504364, + "memory(GiB)": 41.24, + "nll_loss": 0.3030712604522705, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2642664313316345, + "rewards/margins": 7.56298828125, + "rewards/rejected": -7.298722267150879, + "step": 269, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.5535135135135136, + "grad_norm": 1.3857908248901367, + "learning_rate": 0.00010159303233653604, + "logits/chosen": -0.570176899433136, + "logits/rejected": -0.8510745763778687, + "logps/chosen": -2.5669748783111572, + "logps/rejected": -91.87413024902344, + "loss": 0.24940097332000732, + "memory(GiB)": 41.24, + "nll_loss": 0.2209518700838089, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4336439371109009, + "rewards/margins": 8.137587547302246, + "rewards/rejected": -7.703943252563477, + "step": 270, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.5592792792792793, + "grad_norm": 3.1311755180358887, + "learning_rate": 0.00010095584527817319, + "logits/chosen": -0.584690272808075, + "logits/rejected": -0.9461975693702698, + "logps/chosen": -1.4432144165039062, + "logps/rejected": -90.90803527832031, + "loss": 0.17027394473552704, + "memory(GiB)": 41.24, + "nll_loss": 0.15755578875541687, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1908416450023651, + "rewards/margins": 7.701169013977051, + "rewards/rejected": -7.510327339172363, + "step": 271, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.565045045045045, + "grad_norm": 0.8555260300636292, + "learning_rate": 0.00010031861940548555, + "logits/chosen": -0.5870640873908997, + "logits/rejected": -0.8846918940544128, + "logps/chosen": -3.1194140911102295, + "logps/rejected": -84.05816650390625, + "loss": 0.2235853672027588, + "memory(GiB)": 41.24, + "nll_loss": 0.19342783093452454, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32051804661750793, + "rewards/margins": 7.1265106201171875, + "rewards/rejected": -6.805992126464844, + "step": 272, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.5708108108108108, + "grad_norm": 0.4469135105609894, + "learning_rate": 9.968138059451446e-05, + "logits/chosen": -0.561904788017273, + "logits/rejected": -0.9012641310691833, + "logps/chosen": -3.200127363204956, + "logps/rejected": -87.4157943725586, + "loss": 0.2398141324520111, + "memory(GiB)": 41.24, + "nll_loss": 0.21223551034927368, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.25336456298828125, + "rewards/margins": 7.232346534729004, + "rewards/rejected": -6.978981971740723, + "step": 273, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.5765765765765765, + "grad_norm": 1.172140121459961, + "learning_rate": 9.904415472182682e-05, + "logits/chosen": -0.4839547276496887, + "logits/rejected": -0.8654844760894775, + "logps/chosen": -1.2689763307571411, + "logps/rejected": -100.70805358886719, + "loss": 0.13727417588233948, + "memory(GiB)": 41.24, + "nll_loss": 0.08727098256349564, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.45773637294769287, + "rewards/margins": 8.578876495361328, + "rewards/rejected": -8.121140480041504, + "step": 274, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.5823423423423424, + "grad_norm": 1.1815749406814575, + "learning_rate": 9.840696766346401e-05, + "logits/chosen": -0.6512535810470581, + "logits/rejected": -0.8718971610069275, + "logps/chosen": -2.4228830337524414, + "logps/rejected": -83.79597473144531, + "loss": 0.2599056363105774, + "memory(GiB)": 41.24, + "nll_loss": 0.17275774478912354, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.30599865317344666, + "rewards/margins": 7.236001968383789, + "rewards/rejected": -6.930003643035889, + "step": 275, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.588108108108108, + "grad_norm": 1.087708830833435, + "learning_rate": 9.776984529389131e-05, + "logits/chosen": -0.5125508904457092, + "logits/rejected": -0.8501510620117188, + "logps/chosen": -1.924040675163269, + "logps/rejected": -95.89266967773438, + "loss": 0.13055917620658875, + "memory(GiB)": 41.24, + "nll_loss": 0.11521827429533005, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2806589901447296, + "rewards/margins": 7.590930938720703, + "rewards/rejected": -7.310272216796875, + "step": 276, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.593873873873874, + "grad_norm": 0.926349401473999, + "learning_rate": 9.713281348494726e-05, + "logits/chosen": -0.5143563747406006, + "logits/rejected": -0.9174257516860962, + "logps/chosen": -0.9051769971847534, + "logps/rejected": -98.711669921875, + "loss": 0.11217982321977615, + "memory(GiB)": 41.24, + "nll_loss": 0.057560034096241, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14916111528873444, + "rewards/margins": 7.899416446685791, + "rewards/rejected": -7.750256538391113, + "step": 277, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.5996396396396397, + "grad_norm": 1.0033612251281738, + "learning_rate": 9.64958981047928e-05, + "logits/chosen": -0.5166518688201904, + "logits/rejected": -0.8793089389801025, + "logps/chosen": -4.466638565063477, + "logps/rejected": -103.69050598144531, + "loss": 0.3073112368583679, + "memory(GiB)": 41.24, + "nll_loss": 0.2843497693538666, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32026752829551697, + "rewards/margins": 8.899625778198242, + "rewards/rejected": -8.579357147216797, + "step": 278, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6054054054054054, + "grad_norm": 1.239680528640747, + "learning_rate": 9.585912501686111e-05, + "logits/chosen": -0.4977273643016815, + "logits/rejected": -0.8297019004821777, + "logps/chosen": -2.0252184867858887, + "logps/rejected": -93.17738342285156, + "loss": 0.2024317979812622, + "memory(GiB)": 41.24, + "nll_loss": 0.1512202024459839, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.35597655177116394, + "rewards/margins": 7.539138317108154, + "rewards/rejected": -7.183161735534668, + "step": 279, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.6111711711711711, + "grad_norm": 1.532649040222168, + "learning_rate": 9.522252007880732e-05, + "logits/chosen": -0.6540271639823914, + "logits/rejected": -0.9306864738464355, + "logps/chosen": -1.1256556510925293, + "logps/rejected": -96.81440734863281, + "loss": 0.13118696212768555, + "memory(GiB)": 41.24, + "nll_loss": 0.06020975112915039, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.30623379349708557, + "rewards/margins": 8.51535415649414, + "rewards/rejected": -8.20911979675293, + "step": 280, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6169369369369369, + "grad_norm": 1.0006412267684937, + "learning_rate": 9.458610914145826e-05, + "logits/chosen": -0.5862869024276733, + "logits/rejected": -0.9326885938644409, + "logps/chosen": -3.7132813930511475, + "logps/rejected": -95.93994903564453, + "loss": 0.2073487639427185, + "memory(GiB)": 41.24, + "nll_loss": 0.18085618317127228, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3810235559940338, + "rewards/margins": 8.14185619354248, + "rewards/rejected": -7.760832786560059, + "step": 281, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6227027027027026, + "grad_norm": 0.9711495637893677, + "learning_rate": 9.394991804776309e-05, + "logits/chosen": -0.552708089351654, + "logits/rejected": -0.8055617809295654, + "logps/chosen": -2.3333301544189453, + "logps/rejected": -84.41504669189453, + "loss": 0.13116180896759033, + "memory(GiB)": 41.24, + "nll_loss": 0.10828734934329987, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.40816813707351685, + "rewards/margins": 7.26495361328125, + "rewards/rejected": -6.856784820556641, + "step": 282, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6284684684684685, + "grad_norm": 0.6350001692771912, + "learning_rate": 9.331397263174364e-05, + "logits/chosen": -0.4492097496986389, + "logits/rejected": -0.8619819283485413, + "logps/chosen": -1.9441540241241455, + "logps/rejected": -109.03350067138672, + "loss": 0.1653510183095932, + "memory(GiB)": 41.24, + "nll_loss": 0.14039570093154907, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.42938756942749023, + "rewards/margins": 8.919137954711914, + "rewards/rejected": -8.489750862121582, + "step": 283, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.6342342342342342, + "grad_norm": 0.6960185170173645, + "learning_rate": 9.267829871744536e-05, + "logits/chosen": -0.618625283241272, + "logits/rejected": -0.9121880531311035, + "logps/chosen": -1.6954020261764526, + "logps/rejected": -94.96092224121094, + "loss": 0.07639533281326294, + "memory(GiB)": 41.24, + "nll_loss": 0.05519339069724083, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.28837311267852783, + "rewards/margins": 8.253684997558594, + "rewards/rejected": -7.965311527252197, + "step": 284, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.6400000000000001, + "grad_norm": 1.3940598964691162, + "learning_rate": 9.204292211788884e-05, + "logits/chosen": -0.578076183795929, + "logits/rejected": -0.9528270959854126, + "logps/chosen": -1.8992458581924438, + "logps/rejected": -92.11747741699219, + "loss": 0.16904497146606445, + "memory(GiB)": 41.24, + "nll_loss": 0.13233791291713715, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26542994379997253, + "rewards/margins": 7.916376113891602, + "rewards/rejected": -7.650946140289307, + "step": 285, + "train_speed(iter/s)": 0.010729 + }, + { + "epoch": 1.6457657657657658, + "grad_norm": 0.9293711185455322, + "learning_rate": 9.140786863402147e-05, + "logits/chosen": -0.6649171113967896, + "logits/rejected": -0.9644867181777954, + "logps/chosen": -2.1775617599487305, + "logps/rejected": -78.7835693359375, + "loss": 0.19159023463726044, + "memory(GiB)": 41.24, + "nll_loss": 0.1725030243396759, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2781451642513275, + "rewards/margins": 6.788741111755371, + "rewards/rejected": -6.51059627532959, + "step": 286, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6515315315315315, + "grad_norm": 1.1537113189697266, + "learning_rate": 9.077316405366981e-05, + "logits/chosen": -0.6535583138465881, + "logits/rejected": -0.9935054183006287, + "logps/chosen": -3.1851658821105957, + "logps/rejected": -92.34557342529297, + "loss": 0.24425984919071198, + "memory(GiB)": 41.24, + "nll_loss": 0.23288863897323608, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31673336029052734, + "rewards/margins": 8.090534210205078, + "rewards/rejected": -7.773800849914551, + "step": 287, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6572972972972972, + "grad_norm": 2.202179193496704, + "learning_rate": 9.01388341504923e-05, + "logits/chosen": -0.5745314955711365, + "logits/rejected": -0.9523424506187439, + "logps/chosen": -4.3446502685546875, + "logps/rejected": -119.23814392089844, + "loss": 0.2483672946691513, + "memory(GiB)": 41.24, + "nll_loss": 0.20222075283527374, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.33365774154663086, + "rewards/margins": 10.17822265625, + "rewards/rejected": -9.844566345214844, + "step": 288, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.663063063063063, + "grad_norm": 0.6017576456069946, + "learning_rate": 8.950490468293279e-05, + "logits/chosen": -0.6120154857635498, + "logits/rejected": -0.9212927222251892, + "logps/chosen": -2.7989425659179688, + "logps/rejected": -86.53015899658203, + "loss": 0.1996924877166748, + "memory(GiB)": 41.24, + "nll_loss": 0.17490829527378082, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.286940336227417, + "rewards/margins": 7.214242458343506, + "rewards/rejected": -6.927302837371826, + "step": 289, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6688288288288289, + "grad_norm": 0.8434000611305237, + "learning_rate": 8.887140139317454e-05, + "logits/chosen": -0.6056364178657532, + "logits/rejected": -0.9164049029350281, + "logps/chosen": -4.816060543060303, + "logps/rejected": -95.74642944335938, + "loss": 0.2582835555076599, + "memory(GiB)": 41.24, + "nll_loss": 0.2520279884338379, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7545455694198608, + "rewards/margins": 8.554335594177246, + "rewards/rejected": -7.799790382385254, + "step": 290, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6745945945945946, + "grad_norm": 4.302709102630615, + "learning_rate": 8.823835000609482e-05, + "logits/chosen": -0.5834745168685913, + "logits/rejected": -1.0115468502044678, + "logps/chosen": -2.5484602451324463, + "logps/rejected": -108.22164916992188, + "loss": 0.3237055838108063, + "memory(GiB)": 41.24, + "nll_loss": 0.2526852786540985, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12026344239711761, + "rewards/margins": 9.108240127563477, + "rewards/rejected": -8.98797607421875, + "step": 291, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6803603603603605, + "grad_norm": 1.4632378816604614, + "learning_rate": 8.76057762282203e-05, + "logits/chosen": -0.5660637617111206, + "logits/rejected": -1.061202049255371, + "logps/chosen": -2.468435764312744, + "logps/rejected": -112.27369689941406, + "loss": 0.24110177159309387, + "memory(GiB)": 41.24, + "nll_loss": 0.19869710505008698, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07396674156188965, + "rewards/margins": 9.049920082092285, + "rewards/rejected": -8.975954055786133, + "step": 292, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.6861261261261262, + "grad_norm": 1.314245343208313, + "learning_rate": 8.697370574668335e-05, + "logits/chosen": -0.6602072715759277, + "logits/rejected": -0.9728937149047852, + "logps/chosen": -2.3553061485290527, + "logps/rejected": -101.07364654541016, + "loss": 0.20281532406806946, + "memory(GiB)": 41.24, + "nll_loss": 0.17743095755577087, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5697655081748962, + "rewards/margins": 8.947297096252441, + "rewards/rejected": -8.377532005310059, + "step": 293, + "train_speed(iter/s)": 0.01073 + }, + { + "epoch": 1.691891891891892, + "grad_norm": 1.3260091543197632, + "learning_rate": 8.634216422817867e-05, + "logits/chosen": -0.7734063267707825, + "logits/rejected": -0.987154483795166, + "logps/chosen": -4.309966564178467, + "logps/rejected": -78.10920715332031, + "loss": 0.3564646542072296, + "memory(GiB)": 41.24, + "nll_loss": 0.2989146113395691, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2878839373588562, + "rewards/margins": 6.840456008911133, + "rewards/rejected": -6.552572727203369, + "step": 294, + "train_speed(iter/s)": 0.010731 + }, + { + "epoch": 1.6976576576576576, + "grad_norm": 1.6897776126861572, + "learning_rate": 8.57111773179213e-05, + "logits/chosen": -0.6336952447891235, + "logits/rejected": -1.0445548295974731, + "logps/chosen": -1.379799723625183, + "logps/rejected": -93.2787857055664, + "loss": 0.14356985688209534, + "memory(GiB)": 41.24, + "nll_loss": 0.09856417775154114, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23449544608592987, + "rewards/margins": 7.955099582672119, + "rewards/rejected": -7.720603942871094, + "step": 295, + "train_speed(iter/s)": 0.010731 + }, + { + "epoch": 1.7034234234234233, + "grad_norm": 0.9327660799026489, + "learning_rate": 8.508077063860506e-05, + "logits/chosen": -0.5344192981719971, + "logits/rejected": -1.0205243825912476, + "logps/chosen": -0.8863539695739746, + "logps/rejected": -122.19490051269531, + "loss": 0.20819178223609924, + "memory(GiB)": 41.24, + "nll_loss": 0.1362844705581665, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10618732124567032, + "rewards/margins": 9.871743202209473, + "rewards/rejected": -9.765556335449219, + "step": 296, + "train_speed(iter/s)": 0.010731 + }, + { + "epoch": 1.709189189189189, + "grad_norm": 7.680380821228027, + "learning_rate": 8.44509697893621e-05, + "logits/chosen": -0.6720832586288452, + "logits/rejected": -0.9245396852493286, + "logps/chosen": -4.568170070648193, + "logps/rejected": -91.47016906738281, + "loss": 0.3026784062385559, + "memory(GiB)": 41.24, + "nll_loss": 0.2759172320365906, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.28561854362487793, + "rewards/margins": 7.839561939239502, + "rewards/rejected": -7.553943634033203, + "step": 297, + "train_speed(iter/s)": 0.010731 + }, + { + "epoch": 1.714954954954955, + "grad_norm": 1.1304014921188354, + "learning_rate": 8.382180034472353e-05, + "logits/chosen": -0.5577611327171326, + "logits/rejected": -0.9694806337356567, + "logps/chosen": -2.482516288757324, + "logps/rejected": -124.2740478515625, + "loss": 0.17773966491222382, + "memory(GiB)": 41.24, + "nll_loss": 0.16795828938484192, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18939006328582764, + "rewards/margins": 10.547513961791992, + "rewards/rejected": -10.358122825622559, + "step": 298, + "train_speed(iter/s)": 0.010731 + }, + { + "epoch": 1.7207207207207207, + "grad_norm": 3.0718917846679688, + "learning_rate": 8.319328785358078e-05, + "logits/chosen": -0.6541886329650879, + "logits/rejected": -0.9405132532119751, + "logps/chosen": -4.128439903259277, + "logps/rejected": -86.01166534423828, + "loss": 0.2915968596935272, + "memory(GiB)": 41.24, + "nll_loss": 0.28397953510284424, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6267665028572083, + "rewards/margins": 7.876523971557617, + "rewards/rejected": -7.249757289886475, + "step": 299, + "train_speed(iter/s)": 0.010731 + }, + { + "epoch": 1.7264864864864866, + "grad_norm": 1.0582029819488525, + "learning_rate": 8.256545783814802e-05, + "logits/chosen": -0.6806445717811584, + "logits/rejected": -0.9849521517753601, + "logps/chosen": -1.6571637392044067, + "logps/rejected": -89.55918884277344, + "loss": 0.15995512902736664, + "memory(GiB)": 41.24, + "nll_loss": 0.12567465007305145, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3345286250114441, + "rewards/margins": 7.534932613372803, + "rewards/rejected": -7.200402736663818, + "step": 300, + "train_speed(iter/s)": 0.010731 + }, + { + "epoch": 1.7264864864864866, + "eval_logits/chosen": -0.6343019604682922, + "eval_logits/rejected": -0.9546191096305847, + "eval_logps/chosen": -4.193188190460205, + "eval_logps/rejected": -85.89741516113281, + "eval_loss": 0.4250725209712982, + "eval_nll_loss": 0.36724570393562317, + "eval_rewards/accuracies": 0.9642857313156128, + "eval_rewards/chosen": 0.2654857933521271, + "eval_rewards/margins": 7.137094020843506, + "eval_rewards/rejected": -6.871608734130859, + "eval_runtime": 64.4253, + "eval_samples_per_second": 0.869, + "eval_steps_per_second": 0.435, + "step": 300 + }, + { + "epoch": 1.7322522522522523, + "grad_norm": 3.9233813285827637, + "learning_rate": 8.19383357929261e-05, + "logits/chosen": -0.7149097919464111, + "logits/rejected": -0.9879697561264038, + "logps/chosen": -6.41920280456543, + "logps/rejected": -91.33063507080078, + "loss": 0.3479698598384857, + "memory(GiB)": 41.24, + "nll_loss": 0.3068650960922241, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07926599681377411, + "rewards/margins": 8.066815376281738, + "rewards/rejected": -7.987548828125, + "step": 301, + "train_speed(iter/s)": 0.0107 + }, + { + "epoch": 1.738018018018018, + "grad_norm": 0.8075700998306274, + "learning_rate": 8.131194718366696e-05, + "logits/chosen": -0.5910763740539551, + "logits/rejected": -1.0640406608581543, + "logps/chosen": -2.341853380203247, + "logps/rejected": -103.36856079101562, + "loss": 0.49690863490104675, + "memory(GiB)": 41.24, + "nll_loss": 0.3171219825744629, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1597364842891693, + "rewards/margins": 8.580909729003906, + "rewards/rejected": -8.421172142028809, + "step": 302, + "train_speed(iter/s)": 0.0107 + }, + { + "epoch": 1.7437837837837837, + "grad_norm": 7.802829265594482, + "learning_rate": 8.068631744633976e-05, + "logits/chosen": -0.6219963431358337, + "logits/rejected": -0.9677342176437378, + "logps/chosen": -3.086090564727783, + "logps/rejected": -109.29876708984375, + "loss": 0.2125876247882843, + "memory(GiB)": 41.24, + "nll_loss": 0.1926698386669159, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3031369149684906, + "rewards/margins": 9.010135650634766, + "rewards/rejected": -8.706997871398926, + "step": 303, + "train_speed(iter/s)": 0.0107 + }, + { + "epoch": 1.7495495495495494, + "grad_norm": 0.9609875679016113, + "learning_rate": 8.006147198609778e-05, + "logits/chosen": -0.6007803082466125, + "logits/rejected": -0.961830735206604, + "logps/chosen": -1.471971035003662, + "logps/rejected": -102.30226135253906, + "loss": 0.14180457592010498, + "memory(GiB)": 41.24, + "nll_loss": 0.10179757326841354, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3813458979129791, + "rewards/margins": 8.105271339416504, + "rewards/rejected": -7.7239251136779785, + "step": 304, + "train_speed(iter/s)": 0.0107 + }, + { + "epoch": 1.7553153153153152, + "grad_norm": 2.7908036708831787, + "learning_rate": 7.943743617624695e-05, + "logits/chosen": -0.6619476079940796, + "logits/rejected": -0.8973278999328613, + "logps/chosen": -3.9902851581573486, + "logps/rejected": -83.89830780029297, + "loss": 0.28083598613739014, + "memory(GiB)": 41.24, + "nll_loss": 0.24575045704841614, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34188610315322876, + "rewards/margins": 7.177820205688477, + "rewards/rejected": -6.835933685302734, + "step": 305, + "train_speed(iter/s)": 0.0107 + }, + { + "epoch": 1.761081081081081, + "grad_norm": 1.3544881343841553, + "learning_rate": 7.881423535721551e-05, + "logits/chosen": -0.7095592617988586, + "logits/rejected": -1.047359824180603, + "logps/chosen": -1.0230883359909058, + "logps/rejected": -94.56197357177734, + "loss": 0.10650934278964996, + "memory(GiB)": 41.24, + "nll_loss": 0.08602449297904968, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1309710592031479, + "rewards/margins": 7.807618618011475, + "rewards/rejected": -7.676647186279297, + "step": 306, + "train_speed(iter/s)": 0.0107 + }, + { + "epoch": 1.766846846846847, + "grad_norm": 1.6648774147033691, + "learning_rate": 7.819189483552493e-05, + "logits/chosen": -0.5188851952552795, + "logits/rejected": -0.9443877339363098, + "logps/chosen": -2.5106263160705566, + "logps/rejected": -99.06053161621094, + "loss": 0.21319082379341125, + "memory(GiB)": 41.24, + "nll_loss": 0.188232421875, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26766103506088257, + "rewards/margins": 7.974985122680664, + "rewards/rejected": -7.707324028015137, + "step": 307, + "train_speed(iter/s)": 0.0107 + }, + { + "epoch": 1.7726126126126127, + "grad_norm": 1.030945062637329, + "learning_rate": 7.75704398827622e-05, + "logits/chosen": -0.618090808391571, + "logits/rejected": -0.9638433456420898, + "logps/chosen": -2.4359958171844482, + "logps/rejected": -93.35203552246094, + "loss": 0.23479269444942474, + "memory(GiB)": 41.24, + "nll_loss": 0.20363783836364746, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1842065155506134, + "rewards/margins": 7.572452068328857, + "rewards/rejected": -7.388244152069092, + "step": 308, + "train_speed(iter/s)": 0.010701 + }, + { + "epoch": 1.7783783783783784, + "grad_norm": 1.610468864440918, + "learning_rate": 7.694989573455388e-05, + "logits/chosen": -0.6354047060012817, + "logits/rejected": -0.9497014284133911, + "logps/chosen": -2.7137610912323, + "logps/rejected": -90.20038604736328, + "loss": 0.2466818392276764, + "memory(GiB)": 41.24, + "nll_loss": 0.2018105685710907, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19573740661144257, + "rewards/margins": 7.340433120727539, + "rewards/rejected": -7.14469575881958, + "step": 309, + "train_speed(iter/s)": 0.010701 + }, + { + "epoch": 1.7841441441441441, + "grad_norm": 0.8683100342750549, + "learning_rate": 7.633028758954109e-05, + "logits/chosen": -0.4981246292591095, + "logits/rejected": -0.8920284509658813, + "logps/chosen": -1.3449852466583252, + "logps/rejected": -87.27311706542969, + "loss": 0.15985581278800964, + "memory(GiB)": 41.24, + "nll_loss": 0.11629845201969147, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21249940991401672, + "rewards/margins": 6.930314064025879, + "rewards/rejected": -6.717814922332764, + "step": 310, + "train_speed(iter/s)": 0.010701 + }, + { + "epoch": 1.7899099099099098, + "grad_norm": 0.726672887802124, + "learning_rate": 7.571164060835641e-05, + "logits/chosen": -0.5648291707038879, + "logits/rejected": -0.9679074883460999, + "logps/chosen": -0.5762008428573608, + "logps/rejected": -94.06312561035156, + "loss": 0.10999278724193573, + "memory(GiB)": 41.24, + "nll_loss": 0.07327277958393097, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20163661241531372, + "rewards/margins": 7.638485908508301, + "rewards/rejected": -7.4368486404418945, + "step": 311, + "train_speed(iter/s)": 0.010701 + }, + { + "epoch": 1.7956756756756755, + "grad_norm": 0.6270626783370972, + "learning_rate": 7.509397991260202e-05, + "logits/chosen": -0.6544211506843567, + "logits/rejected": -0.999680757522583, + "logps/chosen": -0.8042967915534973, + "logps/rejected": -103.15768432617188, + "loss": 0.0847005620598793, + "memory(GiB)": 41.24, + "nll_loss": 0.07272159308195114, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5687258839607239, + "rewards/margins": 8.942326545715332, + "rewards/rejected": -8.373600959777832, + "step": 312, + "train_speed(iter/s)": 0.010702 + }, + { + "epoch": 1.8014414414414415, + "grad_norm": 0.5516709089279175, + "learning_rate": 7.447733058382981e-05, + "logits/chosen": -0.6301634907722473, + "logits/rejected": -1.0285974740982056, + "logps/chosen": -0.531864583492279, + "logps/rejected": -97.67058563232422, + "loss": 0.08058549463748932, + "memory(GiB)": 41.24, + "nll_loss": 0.06633614003658295, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1902368813753128, + "rewards/margins": 8.120416641235352, + "rewards/rejected": -7.930179595947266, + "step": 313, + "train_speed(iter/s)": 0.010702 + }, + { + "epoch": 1.8072072072072072, + "grad_norm": 2.0297648906707764, + "learning_rate": 7.386171766252274e-05, + "logits/chosen": -0.5949457287788391, + "logits/rejected": -0.9591047167778015, + "logps/chosen": -2.422750234603882, + "logps/rejected": -103.86263275146484, + "loss": 0.2555314302444458, + "memory(GiB)": 41.24, + "nll_loss": 0.20459307730197906, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24491086602210999, + "rewards/margins": 8.553497314453125, + "rewards/rejected": -8.308586120605469, + "step": 314, + "train_speed(iter/s)": 0.010702 + }, + { + "epoch": 1.812972972972973, + "grad_norm": 1.3364982604980469, + "learning_rate": 7.324716614707793e-05, + "logits/chosen": -0.7352499961853027, + "logits/rejected": -1.039025068283081, + "logps/chosen": -1.083723545074463, + "logps/rejected": -94.29521942138672, + "loss": 0.1517622470855713, + "memory(GiB)": 41.24, + "nll_loss": 0.10882994532585144, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15984982252120972, + "rewards/margins": 8.365867614746094, + "rewards/rejected": -8.206018447875977, + "step": 315, + "train_speed(iter/s)": 0.010703 + }, + { + "epoch": 1.8187387387387388, + "grad_norm": 0.6706785559654236, + "learning_rate": 7.263370099279172e-05, + "logits/chosen": -0.5640876889228821, + "logits/rejected": -0.8643637299537659, + "logps/chosen": -3.242088794708252, + "logps/rejected": -88.01509094238281, + "loss": 0.14969868957996368, + "memory(GiB)": 41.24, + "nll_loss": 0.13343137502670288, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.29573485255241394, + "rewards/margins": 7.431912422180176, + "rewards/rejected": -7.1361775398254395, + "step": 316, + "train_speed(iter/s)": 0.010703 + }, + { + "epoch": 1.8245045045045045, + "grad_norm": 0.5097038745880127, + "learning_rate": 7.202134711084624e-05, + "logits/chosen": -0.6677834391593933, + "logits/rejected": -1.0189274549484253, + "logps/chosen": -2.0945234298706055, + "logps/rejected": -105.15029907226562, + "loss": 0.16207458078861237, + "memory(GiB)": 41.24, + "nll_loss": 0.1545000672340393, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6054971218109131, + "rewards/margins": 9.389901161193848, + "rewards/rejected": -8.784403800964355, + "step": 317, + "train_speed(iter/s)": 0.010703 + }, + { + "epoch": 1.8302702702702702, + "grad_norm": 1.082793951034546, + "learning_rate": 7.141012936729771e-05, + "logits/chosen": -0.6712667346000671, + "logits/rejected": -1.0438780784606934, + "logps/chosen": -0.4942001402378082, + "logps/rejected": -105.9577407836914, + "loss": 0.06889337301254272, + "memory(GiB)": 41.24, + "nll_loss": 0.06211407855153084, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21248969435691833, + "rewards/margins": 9.044974327087402, + "rewards/rejected": -8.832484245300293, + "step": 318, + "train_speed(iter/s)": 0.010703 + }, + { + "epoch": 1.836036036036036, + "grad_norm": 1.6099659204483032, + "learning_rate": 7.080007258206698e-05, + "logits/chosen": -0.6598040461540222, + "logits/rejected": -1.0132709741592407, + "logps/chosen": -1.6172504425048828, + "logps/rejected": -103.41716003417969, + "loss": 0.1941622793674469, + "memory(GiB)": 41.24, + "nll_loss": 0.16482065618038177, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1944759637117386, + "rewards/margins": 9.017773628234863, + "rewards/rejected": -8.823297500610352, + "step": 319, + "train_speed(iter/s)": 0.010703 + }, + { + "epoch": 1.8418018018018016, + "grad_norm": 1.1799566745758057, + "learning_rate": 7.019120152793135e-05, + "logits/chosen": -0.679556131362915, + "logits/rejected": -0.9549793004989624, + "logps/chosen": -2.8714540004730225, + "logps/rejected": -98.24635314941406, + "loss": 0.2316533327102661, + "memory(GiB)": 41.24, + "nll_loss": 0.19853028655052185, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24695701897144318, + "rewards/margins": 8.704883575439453, + "rewards/rejected": -8.457927703857422, + "step": 320, + "train_speed(iter/s)": 0.010704 + }, + { + "epoch": 1.8475675675675676, + "grad_norm": 0.7303210496902466, + "learning_rate": 6.95835409295187e-05, + "logits/chosen": -0.6523126363754272, + "logits/rejected": -1.020486831665039, + "logps/chosen": -3.729771614074707, + "logps/rejected": -135.64431762695312, + "loss": 0.14604772627353668, + "memory(GiB)": 41.24, + "nll_loss": 0.14135172963142395, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11208090931177139, + "rewards/margins": 11.833364486694336, + "rewards/rejected": -11.721282958984375, + "step": 321, + "train_speed(iter/s)": 0.010704 + }, + { + "epoch": 1.8533333333333335, + "grad_norm": 0.826923668384552, + "learning_rate": 6.89771154623037e-05, + "logits/chosen": -0.6271218061447144, + "logits/rejected": -1.0214686393737793, + "logps/chosen": -1.974669098854065, + "logps/rejected": -119.37078857421875, + "loss": 0.18596968054771423, + "memory(GiB)": 41.24, + "nll_loss": 0.1433202624320984, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.25551721453666687, + "rewards/margins": 10.3358154296875, + "rewards/rejected": -10.080297470092773, + "step": 322, + "train_speed(iter/s)": 0.010704 + }, + { + "epoch": 1.8590990990990992, + "grad_norm": 1.5200361013412476, + "learning_rate": 6.837194975160554e-05, + "logits/chosen": -0.7221765518188477, + "logits/rejected": -1.0179543495178223, + "logps/chosen": -2.746236801147461, + "logps/rejected": -94.35678100585938, + "loss": 0.1959131509065628, + "memory(GiB)": 41.24, + "nll_loss": 0.16196052730083466, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24598492681980133, + "rewards/margins": 8.168785095214844, + "rewards/rejected": -7.922799587249756, + "step": 323, + "train_speed(iter/s)": 0.010704 + }, + { + "epoch": 1.864864864864865, + "grad_norm": 0.8893440961837769, + "learning_rate": 6.776806837158802e-05, + "logits/chosen": -0.7515994906425476, + "logits/rejected": -1.0873609781265259, + "logps/chosen": -1.9344902038574219, + "logps/rejected": -111.62481689453125, + "loss": 0.17805160582065582, + "memory(GiB)": 41.24, + "nll_loss": 0.14971868693828583, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16122399270534515, + "rewards/margins": 9.678110122680664, + "rewards/rejected": -9.516885757446289, + "step": 324, + "train_speed(iter/s)": 0.010704 + }, + { + "epoch": 1.8706306306306306, + "grad_norm": 0.7393608689308167, + "learning_rate": 6.716549584426182e-05, + "logits/chosen": -0.6963468790054321, + "logits/rejected": -1.0317466259002686, + "logps/chosen": -2.897395610809326, + "logps/rejected": -101.7484130859375, + "loss": 0.24223023653030396, + "memory(GiB)": 41.24, + "nll_loss": 0.23367911577224731, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26319146156311035, + "rewards/margins": 8.788366317749023, + "rewards/rejected": -8.525175094604492, + "step": 325, + "train_speed(iter/s)": 0.010705 + }, + { + "epoch": 1.8763963963963963, + "grad_norm": 1.7007874250411987, + "learning_rate": 6.656425663848848e-05, + "logits/chosen": -0.6554928421974182, + "logits/rejected": -0.9848640561103821, + "logps/chosen": -2.715691328048706, + "logps/rejected": -103.58108520507812, + "loss": 0.26388633251190186, + "memory(GiB)": 41.24, + "nll_loss": 0.23348112404346466, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3149644136428833, + "rewards/margins": 9.236834526062012, + "rewards/rejected": -8.921870231628418, + "step": 326, + "train_speed(iter/s)": 0.010705 + }, + { + "epoch": 1.882162162162162, + "grad_norm": 0.5721933245658875, + "learning_rate": 6.596437516898703e-05, + "logits/chosen": -0.7064806222915649, + "logits/rejected": -1.0539876222610474, + "logps/chosen": -2.050990104675293, + "logps/rejected": -126.16415405273438, + "loss": 0.11638378351926804, + "memory(GiB)": 41.24, + "nll_loss": 0.08542951941490173, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.0660579577088356, + "rewards/margins": 10.927128791809082, + "rewards/rejected": -10.861071586608887, + "step": 327, + "train_speed(iter/s)": 0.010705 + }, + { + "epoch": 1.887927927927928, + "grad_norm": 1.6554800271987915, + "learning_rate": 6.536587579534236e-05, + "logits/chosen": -0.7383469343185425, + "logits/rejected": -1.1014130115509033, + "logps/chosen": -1.1747007369995117, + "logps/rejected": -127.77891540527344, + "loss": 0.07589925080537796, + "memory(GiB)": 41.24, + "nll_loss": 0.06867410242557526, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26423317193984985, + "rewards/margins": 11.404837608337402, + "rewards/rejected": -11.140604019165039, + "step": 328, + "train_speed(iter/s)": 0.010705 + }, + { + "epoch": 1.8936936936936937, + "grad_norm": 5.660789489746094, + "learning_rate": 6.476878282101614e-05, + "logits/chosen": -0.614322304725647, + "logits/rejected": -1.1342331171035767, + "logps/chosen": -3.220783233642578, + "logps/rejected": -169.4235382080078, + "loss": 0.2335151582956314, + "memory(GiB)": 41.24, + "nll_loss": 0.21671630442142487, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.04895550757646561, + "rewards/margins": 14.301774978637695, + "rewards/rejected": -14.350728988647461, + "step": 329, + "train_speed(iter/s)": 0.010705 + }, + { + "epoch": 1.8994594594594596, + "grad_norm": 0.3728070855140686, + "learning_rate": 6.417312049236004e-05, + "logits/chosen": -0.5540862679481506, + "logits/rejected": -1.0590349435806274, + "logps/chosen": -1.2624342441558838, + "logps/rejected": -132.83248901367188, + "loss": 0.11809687316417694, + "memory(GiB)": 41.24, + "nll_loss": 0.09898046404123306, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31623268127441406, + "rewards/margins": 11.09921932220459, + "rewards/rejected": -10.782986640930176, + "step": 330, + "train_speed(iter/s)": 0.010705 + }, + { + "epoch": 1.9052252252252253, + "grad_norm": 0.8762845993041992, + "learning_rate": 6.357891299763086e-05, + "logits/chosen": -0.69197678565979, + "logits/rejected": -0.9935667514801025, + "logps/chosen": -4.758241653442383, + "logps/rejected": -106.16485595703125, + "loss": 0.300553560256958, + "memory(GiB)": 41.24, + "nll_loss": 0.27140897512435913, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3382798731327057, + "rewards/margins": 9.438005447387695, + "rewards/rejected": -9.099724769592285, + "step": 331, + "train_speed(iter/s)": 0.010705 + }, + { + "epoch": 1.910990990990991, + "grad_norm": 2.496192693710327, + "learning_rate": 6.298618446600856e-05, + "logits/chosen": -0.7820520997047424, + "logits/rejected": -0.9762202501296997, + "logps/chosen": -2.9556801319122314, + "logps/rejected": -81.24717712402344, + "loss": 0.26701855659484863, + "memory(GiB)": 41.24, + "nll_loss": 0.18815891444683075, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.3220246434211731, + "rewards/margins": 7.285861015319824, + "rewards/rejected": -6.963836669921875, + "step": 332, + "train_speed(iter/s)": 0.010706 + }, + { + "epoch": 1.9167567567567567, + "grad_norm": 1.1889234781265259, + "learning_rate": 6.239495896661643e-05, + "logits/chosen": -0.6824884414672852, + "logits/rejected": -1.0354845523834229, + "logps/chosen": -1.5120099782943726, + "logps/rejected": -119.75020599365234, + "loss": 0.16994014382362366, + "memory(GiB)": 41.24, + "nll_loss": 0.1452503502368927, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4305982291698456, + "rewards/margins": 10.441259384155273, + "rewards/rejected": -10.010661125183105, + "step": 333, + "train_speed(iter/s)": 0.010706 + }, + { + "epoch": 1.9225225225225224, + "grad_norm": 0.5564529895782471, + "learning_rate": 6.180526050754346e-05, + "logits/chosen": -0.746012270450592, + "logits/rejected": -1.0110886096954346, + "logps/chosen": -1.5846396684646606, + "logps/rejected": -89.84294128417969, + "loss": 0.12058084458112717, + "memory(GiB)": 41.24, + "nll_loss": 0.10888123512268066, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5301511287689209, + "rewards/margins": 7.960659980773926, + "rewards/rejected": -7.430509090423584, + "step": 334, + "train_speed(iter/s)": 0.010706 + }, + { + "epoch": 1.9282882882882881, + "grad_norm": 1.4427258968353271, + "learning_rate": 6.12171130348698e-05, + "logits/chosen": -0.6507196426391602, + "logits/rejected": -1.0314421653747559, + "logps/chosen": -2.498617172241211, + "logps/rejected": -117.58732604980469, + "loss": 0.20319777727127075, + "memory(GiB)": 41.24, + "nll_loss": 0.13672159612178802, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.23165585100650787, + "rewards/margins": 9.987587928771973, + "rewards/rejected": -9.755931854248047, + "step": 335, + "train_speed(iter/s)": 0.010706 + }, + { + "epoch": 1.934054054054054, + "grad_norm": 0.7429758906364441, + "learning_rate": 6.0630540431694026e-05, + "logits/chosen": -0.663485050201416, + "logits/rejected": -1.0316194295883179, + "logps/chosen": -1.0613189935684204, + "logps/rejected": -118.62425231933594, + "loss": 0.14050814509391785, + "memory(GiB)": 41.24, + "nll_loss": 0.11347251385450363, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26233720779418945, + "rewards/margins": 10.100933074951172, + "rewards/rejected": -9.838595390319824, + "step": 336, + "train_speed(iter/s)": 0.010706 + }, + { + "epoch": 1.9398198198198198, + "grad_norm": 1.0151944160461426, + "learning_rate": 6.004556651716354e-05, + "logits/chosen": -0.6722368001937866, + "logits/rejected": -0.9373372793197632, + "logps/chosen": -3.6936416625976562, + "logps/rejected": -101.26123809814453, + "loss": 0.208326518535614, + "memory(GiB)": 41.24, + "nll_loss": 0.18723271787166595, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.37735119462013245, + "rewards/margins": 8.75652027130127, + "rewards/rejected": -8.379169464111328, + "step": 337, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 1.9455855855855857, + "grad_norm": 17.15276336669922, + "learning_rate": 5.946221504550732e-05, + "logits/chosen": -0.7422499656677246, + "logits/rejected": -1.0179252624511719, + "logps/chosen": -2.4305574893951416, + "logps/rejected": -96.93527221679688, + "loss": 0.24250547587871552, + "memory(GiB)": 41.24, + "nll_loss": 0.20494799315929413, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4128715991973877, + "rewards/margins": 8.494330406188965, + "rewards/rejected": -8.081459045410156, + "step": 338, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 1.9513513513513514, + "grad_norm": 0.9688490033149719, + "learning_rate": 5.888050970507114e-05, + "logits/chosen": -0.5074092149734497, + "logits/rejected": -0.992556095123291, + "logps/chosen": -3.6099486351013184, + "logps/rejected": -123.67737579345703, + "loss": 0.26563432812690735, + "memory(GiB)": 41.24, + "nll_loss": 0.24966250360012054, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22994372248649597, + "rewards/margins": 10.088909149169922, + "rewards/rejected": -9.858965873718262, + "step": 339, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 1.9571171171171171, + "grad_norm": 2.4413914680480957, + "learning_rate": 5.830047411735588e-05, + "logits/chosen": -0.670777440071106, + "logits/rejected": -0.9375532865524292, + "logps/chosen": -4.536951541900635, + "logps/rejected": -92.39743041992188, + "loss": 0.23023152351379395, + "memory(GiB)": 41.24, + "nll_loss": 0.21488457918167114, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23933574557304382, + "rewards/margins": 7.656829357147217, + "rewards/rejected": -7.41749382019043, + "step": 340, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 1.9628828828828828, + "grad_norm": 1.0596953630447388, + "learning_rate": 5.772213183605817e-05, + "logits/chosen": -0.6010651588439941, + "logits/rejected": -1.0178173780441284, + "logps/chosen": -1.717962622642517, + "logps/rejected": -132.110107421875, + "loss": 0.12807117402553558, + "memory(GiB)": 41.24, + "nll_loss": 0.11598215997219086, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2744857966899872, + "rewards/margins": 11.212226867675781, + "rewards/rejected": -10.93774127960205, + "step": 341, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 1.9686486486486485, + "grad_norm": 1.389633059501648, + "learning_rate": 5.714550634611401e-05, + "logits/chosen": -0.7146607637405396, + "logits/rejected": -1.0605430603027344, + "logps/chosen": -0.6846324801445007, + "logps/rejected": -109.4912109375, + "loss": 0.06890799850225449, + "memory(GiB)": 41.24, + "nll_loss": 0.058928776532411575, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23026201128959656, + "rewards/margins": 9.68777847290039, + "rewards/rejected": -9.457515716552734, + "step": 342, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 1.9744144144144145, + "grad_norm": 0.9664366841316223, + "learning_rate": 5.6570621062745146e-05, + "logits/chosen": -0.6192926168441772, + "logits/rejected": -0.9919301271438599, + "logps/chosen": -3.9573330879211426, + "logps/rejected": -110.93566131591797, + "loss": 0.2838752865791321, + "memory(GiB)": 41.24, + "nll_loss": 0.2563166916370392, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3844650089740753, + "rewards/margins": 9.36357593536377, + "rewards/rejected": -8.979110717773438, + "step": 343, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 1.9801801801801802, + "grad_norm": 1.2836169004440308, + "learning_rate": 5.5997499330508066e-05, + "logits/chosen": -0.6558486223220825, + "logits/rejected": -0.9970208406448364, + "logps/chosen": -5.186588287353516, + "logps/rejected": -105.42745971679688, + "loss": 0.3528812527656555, + "memory(GiB)": 41.24, + "nll_loss": 0.33754995465278625, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22177468240261078, + "rewards/margins": 8.897873878479004, + "rewards/rejected": -8.67609977722168, + "step": 344, + "train_speed(iter/s)": 0.010707 + }, + { + "epoch": 1.985945945945946, + "grad_norm": 0.7625303864479065, + "learning_rate": 5.542616442234618e-05, + "logits/chosen": -0.7527819871902466, + "logits/rejected": -0.9739503860473633, + "logps/chosen": -5.119755744934082, + "logps/rejected": -97.09973907470703, + "loss": 0.2499479502439499, + "memory(GiB)": 41.24, + "nll_loss": 0.2005118429660797, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.30508890748023987, + "rewards/margins": 7.9966044425964355, + "rewards/rejected": -7.69151496887207, + "step": 345, + "train_speed(iter/s)": 0.010708 + }, + { + "epoch": 1.9917117117117118, + "grad_norm": 0.7444700598716736, + "learning_rate": 5.485663953864484e-05, + "logits/chosen": -0.6517712473869324, + "logits/rejected": -0.9245800971984863, + "logps/chosen": -3.4297103881835938, + "logps/rejected": -111.82066345214844, + "loss": 0.1872328221797943, + "memory(GiB)": 41.24, + "nll_loss": 0.16381250321865082, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.395004540681839, + "rewards/margins": 9.727705001831055, + "rewards/rejected": -9.332700729370117, + "step": 346, + "train_speed(iter/s)": 0.010708 + }, + { + "epoch": 1.9974774774774775, + "grad_norm": 0.7278358340263367, + "learning_rate": 5.428894780628899e-05, + "logits/chosen": -0.6262625455856323, + "logits/rejected": -1.0132476091384888, + "logps/chosen": -2.7599048614501953, + "logps/rejected": -128.29429626464844, + "loss": 0.14804692566394806, + "memory(GiB)": 41.24, + "nll_loss": 0.1370442658662796, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12297943234443665, + "rewards/margins": 10.652461051940918, + "rewards/rejected": -10.529480934143066, + "step": 347, + "train_speed(iter/s)": 0.010708 + }, + { + "epoch": 2.0, + "grad_norm": 0.7278358340263367, + "learning_rate": 5.372311227772431e-05, + "logits/chosen": -0.6291365027427673, + "logits/rejected": -1.1446304321289062, + "logps/chosen": -3.447653944022022e-05, + "logps/rejected": -146.0049285888672, + "loss": 0.0017077529337257147, + "memory(GiB)": 41.24, + "nll_loss": 5.74608975512092e-06, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11583631485700607, + "rewards/margins": 12.548344612121582, + "rewards/rejected": -12.43250846862793, + "step": 348, + "train_speed(iter/s)": 0.010726 + }, + { + "epoch": 2.0057657657657657, + "grad_norm": 2.2626681327819824, + "learning_rate": 5.3159155930021e-05, + "logits/chosen": -0.7636749744415283, + "logits/rejected": -1.0101430416107178, + "logps/chosen": -1.7633554935455322, + "logps/rejected": -102.3580093383789, + "loss": 0.1495026797056198, + "memory(GiB)": 41.24, + "nll_loss": 0.13758480548858643, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.551813006401062, + "rewards/margins": 9.395806312561035, + "rewards/rejected": -8.843993186950684, + "step": 349, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 2.0115315315315314, + "grad_norm": 0.7532277703285217, + "learning_rate": 5.259710166394062e-05, + "logits/chosen": -0.7781978249549866, + "logits/rejected": -1.0975528955459595, + "logps/chosen": -0.8418835401535034, + "logps/rejected": -111.32673645019531, + "loss": 0.08846607804298401, + "memory(GiB)": 41.24, + "nll_loss": 0.07249496132135391, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3656042814254761, + "rewards/margins": 9.965299606323242, + "rewards/rejected": -9.599695205688477, + "step": 350, + "train_speed(iter/s)": 0.010725 + }, + { + "epoch": 2.017297297297297, + "grad_norm": 0.5835037231445312, + "learning_rate": 5.2036972303006426e-05, + "logits/chosen": -0.666458010673523, + "logits/rejected": -1.0365673303604126, + "logps/chosen": -2.565896511077881, + "logps/rejected": -110.77236938476562, + "loss": 0.22194571793079376, + "memory(GiB)": 41.24, + "nll_loss": 0.2166982740163803, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27980050444602966, + "rewards/margins": 9.695403099060059, + "rewards/rejected": -9.415603637695312, + "step": 351, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.0230630630630633, + "grad_norm": 4.682882785797119, + "learning_rate": 5.147879059257632e-05, + "logits/chosen": -0.7170751690864563, + "logits/rejected": -1.0876530408859253, + "logps/chosen": -1.1384719610214233, + "logps/rejected": -117.07299041748047, + "loss": 0.04353953152894974, + "memory(GiB)": 41.24, + "nll_loss": 0.036060184240341187, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24029874801635742, + "rewards/margins": 10.151776313781738, + "rewards/rejected": -9.911477088928223, + "step": 352, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.028828828828829, + "grad_norm": 0.202779620885849, + "learning_rate": 5.092257919891929e-05, + "logits/chosen": -0.6059564352035522, + "logits/rejected": -0.9432659149169922, + "logps/chosen": -1.381291389465332, + "logps/rejected": -113.65862274169922, + "loss": 0.13137969374656677, + "memory(GiB)": 41.24, + "nll_loss": 0.0688488632440567, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.37236571311950684, + "rewards/margins": 9.848228454589844, + "rewards/rejected": -9.475863456726074, + "step": 353, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.0345945945945947, + "grad_norm": 0.5466320514678955, + "learning_rate": 5.036836070829512e-05, + "logits/chosen": -0.69258052110672, + "logits/rejected": -1.0133129358291626, + "logps/chosen": -1.2574284076690674, + "logps/rejected": -116.70516967773438, + "loss": 0.06431526690721512, + "memory(GiB)": 41.24, + "nll_loss": 0.056370802223682404, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3451860547065735, + "rewards/margins": 10.030974388122559, + "rewards/rejected": -9.68578815460205, + "step": 354, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.0403603603603604, + "grad_norm": 0.3935391306877136, + "learning_rate": 4.98161576260371e-05, + "logits/chosen": -0.6817679405212402, + "logits/rejected": -1.0860364437103271, + "logps/chosen": -1.0361964702606201, + "logps/rejected": -121.82405090332031, + "loss": 0.06941640377044678, + "memory(GiB)": 41.24, + "nll_loss": 0.0657801553606987, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.42753589153289795, + "rewards/margins": 10.532970428466797, + "rewards/rejected": -10.10543441772461, + "step": 355, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.046126126126126, + "grad_norm": 0.42544692754745483, + "learning_rate": 4.926599237563807e-05, + "logits/chosen": -0.7991234660148621, + "logits/rejected": -1.067280888557434, + "logps/chosen": -0.6973099708557129, + "logps/rejected": -87.47538757324219, + "loss": 0.09407985955476761, + "memory(GiB)": 41.24, + "nll_loss": 0.07323485612869263, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.25911667943000793, + "rewards/margins": 7.751730918884277, + "rewards/rejected": -7.492613792419434, + "step": 356, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.051891891891892, + "grad_norm": 0.6260484457015991, + "learning_rate": 4.87178872978401e-05, + "logits/chosen": -0.7503050565719604, + "logits/rejected": -1.0410758256912231, + "logps/chosen": -1.3630163669586182, + "logps/rejected": -100.85547637939453, + "loss": 0.14041030406951904, + "memory(GiB)": 41.24, + "nll_loss": 0.11562740802764893, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4105437099933624, + "rewards/margins": 9.023932456970215, + "rewards/rejected": -8.61338996887207, + "step": 357, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.0576576576576575, + "grad_norm": 0.5852023959159851, + "learning_rate": 4.817186464972702e-05, + "logits/chosen": -0.7689077854156494, + "logits/rejected": -1.066072940826416, + "logps/chosen": -1.0851335525512695, + "logps/rejected": -110.11882781982422, + "loss": 0.06693820655345917, + "memory(GiB)": 41.24, + "nll_loss": 0.055349480360746384, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.36645475029945374, + "rewards/margins": 9.64514446258545, + "rewards/rejected": -9.27868938446045, + "step": 358, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.063423423423423, + "grad_norm": 0.45909425616264343, + "learning_rate": 4.762794660382086e-05, + "logits/chosen": -0.7381240725517273, + "logits/rejected": -1.0753943920135498, + "logps/chosen": -1.6246895790100098, + "logps/rejected": -105.39446258544922, + "loss": 0.12985529005527496, + "memory(GiB)": 41.24, + "nll_loss": 0.10251322388648987, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32686948776245117, + "rewards/margins": 9.259685516357422, + "rewards/rejected": -8.932815551757812, + "step": 359, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.0691891891891894, + "grad_norm": 0.5620059370994568, + "learning_rate": 4.708615524718128e-05, + "logits/chosen": -0.7564532160758972, + "logits/rejected": -1.0099018812179565, + "logps/chosen": -2.1382956504821777, + "logps/rejected": -105.76231384277344, + "loss": 0.10803371667861938, + "memory(GiB)": 41.24, + "nll_loss": 0.08915401250123978, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.40129464864730835, + "rewards/margins": 9.346211433410645, + "rewards/rejected": -8.944916725158691, + "step": 360, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.074954954954955, + "grad_norm": 0.5350908041000366, + "learning_rate": 4.6546512580508804e-05, + "logits/chosen": -0.7144227027893066, + "logits/rejected": -1.0969374179840088, + "logps/chosen": -1.744510293006897, + "logps/rejected": -130.87229919433594, + "loss": 0.10718417167663574, + "memory(GiB)": 41.24, + "nll_loss": 0.1035425141453743, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3886975347995758, + "rewards/margins": 11.342401504516602, + "rewards/rejected": -10.953703880310059, + "step": 361, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.080720720720721, + "grad_norm": 0.5474323630332947, + "learning_rate": 4.600904051725148e-05, + "logits/chosen": -0.7471128702163696, + "logits/rejected": -1.124627709388733, + "logps/chosen": -1.108418583869934, + "logps/rejected": -125.03573608398438, + "loss": 0.08036281168460846, + "memory(GiB)": 41.24, + "nll_loss": 0.07435505092144012, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3394859731197357, + "rewards/margins": 11.102348327636719, + "rewards/rejected": -10.762863159179688, + "step": 362, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.0864864864864865, + "grad_norm": 0.5023143887519836, + "learning_rate": 4.54737608827148e-05, + "logits/chosen": -0.592808723449707, + "logits/rejected": -1.0072365999221802, + "logps/chosen": -1.3401683568954468, + "logps/rejected": -113.59574127197266, + "loss": 0.09403908997774124, + "memory(GiB)": 41.24, + "nll_loss": 0.08931541442871094, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3548210859298706, + "rewards/margins": 9.633330345153809, + "rewards/rejected": -9.278509140014648, + "step": 363, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.092252252252252, + "grad_norm": 0.5266364216804504, + "learning_rate": 4.4940695413175626e-05, + "logits/chosen": -0.8366541266441345, + "logits/rejected": -1.162805199623108, + "logps/chosen": -1.3807404041290283, + "logps/rejected": -125.22923278808594, + "loss": 0.10916555672883987, + "memory(GiB)": 41.24, + "nll_loss": 0.10021603107452393, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34392741322517395, + "rewards/margins": 10.811279296875, + "rewards/rejected": -10.467350959777832, + "step": 364, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.098018018018018, + "grad_norm": 0.8260918855667114, + "learning_rate": 4.440986575499956e-05, + "logits/chosen": -0.8122618198394775, + "logits/rejected": -1.1245052814483643, + "logps/chosen": -3.4402499198913574, + "logps/rejected": -107.24191284179688, + "loss": 0.1661069393157959, + "memory(GiB)": 41.24, + "nll_loss": 0.16078853607177734, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5291856527328491, + "rewards/margins": 9.46258544921875, + "rewards/rejected": -8.93340015411377, + "step": 365, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.1037837837837836, + "grad_norm": 0.6463497281074524, + "learning_rate": 4.388129346376178e-05, + "logits/chosen": -0.7070938944816589, + "logits/rejected": -1.043674111366272, + "logps/chosen": -1.7142434120178223, + "logps/rejected": -115.11770629882812, + "loss": 0.14735746383666992, + "memory(GiB)": 41.24, + "nll_loss": 0.13991089165210724, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5351644158363342, + "rewards/margins": 10.220780372619629, + "rewards/rejected": -9.685615539550781, + "step": 366, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.1095495495495498, + "grad_norm": 0.8617780804634094, + "learning_rate": 4.335500000337189e-05, + "logits/chosen": -0.7722818851470947, + "logits/rejected": -1.114637851715088, + "logps/chosen": -1.6027175188064575, + "logps/rejected": -92.61148834228516, + "loss": 0.1343543827533722, + "memory(GiB)": 41.24, + "nll_loss": 0.12890587747097015, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.49232959747314453, + "rewards/margins": 8.09817123413086, + "rewards/rejected": -7.605842113494873, + "step": 367, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.1153153153153155, + "grad_norm": 0.6545007824897766, + "learning_rate": 4.283100674520219e-05, + "logits/chosen": -0.6476003527641296, + "logits/rejected": -1.0922412872314453, + "logps/chosen": -1.229990005493164, + "logps/rejected": -122.7174072265625, + "loss": 0.11847774684429169, + "memory(GiB)": 41.24, + "nll_loss": 0.0991000235080719, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1961318999528885, + "rewards/margins": 10.45052719116211, + "rewards/rejected": -10.25439453125, + "step": 368, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.121081081081081, + "grad_norm": 0.76774662733078, + "learning_rate": 4.2309334967219995e-05, + "logits/chosen": -0.771206259727478, + "logits/rejected": -1.2116535902023315, + "logps/chosen": -0.48549365997314453, + "logps/rejected": -126.31437683105469, + "loss": 0.05150448530912399, + "memory(GiB)": 41.24, + "nll_loss": 0.03617728501558304, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22985771298408508, + "rewards/margins": 11.005145072937012, + "rewards/rejected": -10.775286674499512, + "step": 369, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.126846846846847, + "grad_norm": 0.5215669870376587, + "learning_rate": 4.179000585312347e-05, + "logits/chosen": -0.7696986198425293, + "logits/rejected": -1.1471494436264038, + "logps/chosen": -2.347403049468994, + "logps/rejected": -106.55985260009766, + "loss": 0.16343547403812408, + "memory(GiB)": 41.24, + "nll_loss": 0.16055680811405182, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6039815545082092, + "rewards/margins": 9.347000122070312, + "rewards/rejected": -8.743019104003906, + "step": 370, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.1326126126126126, + "grad_norm": 1.3849904537200928, + "learning_rate": 4.127304049148142e-05, + "logits/chosen": -0.800149142742157, + "logits/rejected": -1.2087985277175903, + "logps/chosen": -2.788853645324707, + "logps/rejected": -138.6233673095703, + "loss": 0.1621292382478714, + "memory(GiB)": 41.24, + "nll_loss": 0.15882165729999542, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3479587435722351, + "rewards/margins": 12.34463882446289, + "rewards/rejected": -11.996681213378906, + "step": 371, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.1383783783783783, + "grad_norm": 1.0871050357818604, + "learning_rate": 4.0758459874876954e-05, + "logits/chosen": -0.7925397157669067, + "logits/rejected": -1.2393711805343628, + "logps/chosen": -0.6382302045822144, + "logps/rejected": -146.68447875976562, + "loss": 0.04796387627720833, + "memory(GiB)": 41.24, + "nll_loss": 0.046931177377700806, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24639029800891876, + "rewards/margins": 13.184242248535156, + "rewards/rejected": -12.93785285949707, + "step": 372, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.144144144144144, + "grad_norm": 0.6231171488761902, + "learning_rate": 4.024628489905517e-05, + "logits/chosen": -0.8548545837402344, + "logits/rejected": -1.2258063554763794, + "logps/chosen": -1.1896212100982666, + "logps/rejected": -111.88992309570312, + "loss": 0.1278052181005478, + "memory(GiB)": 41.24, + "nll_loss": 0.09388123452663422, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2702849507331848, + "rewards/margins": 9.93159294128418, + "rewards/rejected": -9.661309242248535, + "step": 373, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.1499099099099097, + "grad_norm": 0.7377911806106567, + "learning_rate": 3.973653636207437e-05, + "logits/chosen": -0.7593305706977844, + "logits/rejected": -1.1127400398254395, + "logps/chosen": -1.7194164991378784, + "logps/rejected": -128.9912109375, + "loss": 0.11936316639184952, + "memory(GiB)": 41.24, + "nll_loss": 0.11392650008201599, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3014066517353058, + "rewards/margins": 11.520092010498047, + "rewards/rejected": -11.2186861038208, + "step": 374, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.155675675675676, + "grad_norm": 0.747080385684967, + "learning_rate": 3.9229234963461766e-05, + "logits/chosen": -0.8068004846572876, + "logits/rejected": -1.183035135269165, + "logps/chosen": -1.0765832662582397, + "logps/rejected": -131.47979736328125, + "loss": 0.10721767693758011, + "memory(GiB)": 41.24, + "nll_loss": 0.08280418813228607, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22776326537132263, + "rewards/margins": 11.514312744140625, + "rewards/rejected": -11.286548614501953, + "step": 375, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.1614414414414416, + "grad_norm": 0.6352575421333313, + "learning_rate": 3.872440130337281e-05, + "logits/chosen": -0.8328331708908081, + "logits/rejected": -1.1570136547088623, + "logps/chosen": -1.8065824508666992, + "logps/rejected": -124.5712890625, + "loss": 0.14996710419654846, + "memory(GiB)": 41.24, + "nll_loss": 0.14042823016643524, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5322981476783752, + "rewards/margins": 11.360990524291992, + "rewards/rejected": -10.828693389892578, + "step": 376, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.1672072072072073, + "grad_norm": 1.6398695707321167, + "learning_rate": 3.822205588175457e-05, + "logits/chosen": -0.8589658737182617, + "logits/rejected": -1.191556453704834, + "logps/chosen": -1.7244662046432495, + "logps/rejected": -123.51134490966797, + "loss": 0.09479396790266037, + "memory(GiB)": 41.24, + "nll_loss": 0.08205929398536682, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3387194871902466, + "rewards/margins": 10.990225791931152, + "rewards/rejected": -10.651506423950195, + "step": 377, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.172972972972973, + "grad_norm": 0.6755312085151672, + "learning_rate": 3.772221909751353e-05, + "logits/chosen": -0.7132154703140259, + "logits/rejected": -1.2062201499938965, + "logps/chosen": -0.6907561421394348, + "logps/rejected": -148.38026428222656, + "loss": 0.07819781452417374, + "memory(GiB)": 41.24, + "nll_loss": 0.059374138712882996, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2403748333454132, + "rewards/margins": 12.885276794433594, + "rewards/rejected": -12.644901275634766, + "step": 378, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.1787387387387387, + "grad_norm": 0.6550033688545227, + "learning_rate": 3.722491124768702e-05, + "logits/chosen": -0.8304086923599243, + "logits/rejected": -1.178147554397583, + "logps/chosen": -1.7724758386611938, + "logps/rejected": -124.11712646484375, + "loss": 0.11583742499351501, + "memory(GiB)": 41.24, + "nll_loss": 0.11279667913913727, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8343939781188965, + "rewards/margins": 11.318742752075195, + "rewards/rejected": -10.484349250793457, + "step": 379, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.1845045045045044, + "grad_norm": 0.6172549724578857, + "learning_rate": 3.673015252661909e-05, + "logits/chosen": -0.8117644786834717, + "logits/rejected": -1.2567986249923706, + "logps/chosen": -1.06840181350708, + "logps/rejected": -132.57460021972656, + "loss": 0.11955217272043228, + "memory(GiB)": 41.24, + "nll_loss": 0.11348409205675125, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.265759140253067, + "rewards/margins": 12.054954528808594, + "rewards/rejected": -11.789194107055664, + "step": 380, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.19027027027027, + "grad_norm": 0.8900021910667419, + "learning_rate": 3.623796302514051e-05, + "logits/chosen": -0.926662266254425, + "logits/rejected": -1.2480473518371582, + "logps/chosen": -1.494692087173462, + "logps/rejected": -131.55877685546875, + "loss": 0.11130554229021072, + "memory(GiB)": 41.24, + "nll_loss": 0.10723792761564255, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.40062400698661804, + "rewards/margins": 12.140097618103027, + "rewards/rejected": -11.739474296569824, + "step": 381, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.196036036036036, + "grad_norm": 0.7763493657112122, + "learning_rate": 3.574836272975293e-05, + "logits/chosen": -0.8530454039573669, + "logits/rejected": -1.2138737440109253, + "logps/chosen": -1.330696702003479, + "logps/rejected": -119.04615020751953, + "loss": 0.11612934619188309, + "memory(GiB)": 41.24, + "nll_loss": 0.11148416250944138, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3111475110054016, + "rewards/margins": 10.730779647827148, + "rewards/rejected": -10.419631958007812, + "step": 382, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.201801801801802, + "grad_norm": 0.5878583788871765, + "learning_rate": 3.5261371521817244e-05, + "logits/chosen": -0.8274633288383484, + "logits/rejected": -1.2182953357696533, + "logps/chosen": -1.0751512050628662, + "logps/rejected": -149.44488525390625, + "loss": 0.07359476387500763, + "memory(GiB)": 41.24, + "nll_loss": 0.05785337835550308, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.36692014336586, + "rewards/margins": 13.62150764465332, + "rewards/rejected": -13.254587173461914, + "step": 383, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.2075675675675677, + "grad_norm": 0.43511688709259033, + "learning_rate": 3.47770091767462e-05, + "logits/chosen": -0.884621798992157, + "logits/rejected": -1.2757821083068848, + "logps/chosen": -1.7317819595336914, + "logps/rejected": -121.42090606689453, + "loss": 0.1251467615365982, + "memory(GiB)": 41.24, + "nll_loss": 0.12321247160434723, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.39998066425323486, + "rewards/margins": 10.98726749420166, + "rewards/rejected": -10.587286949157715, + "step": 384, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.2133333333333334, + "grad_norm": 0.704433023929596, + "learning_rate": 3.4295295363201476e-05, + "logits/chosen": -0.8126075863838196, + "logits/rejected": -1.2107995748519897, + "logps/chosen": -2.574423313140869, + "logps/rejected": -122.31694030761719, + "loss": 0.18911413848400116, + "memory(GiB)": 41.24, + "nll_loss": 0.17396005988121033, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.33698683977127075, + "rewards/margins": 11.030517578125, + "rewards/rejected": -10.693531036376953, + "step": 385, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.219099099099099, + "grad_norm": 0.9555876851081848, + "learning_rate": 3.381624964229504e-05, + "logits/chosen": -0.7917225956916809, + "logits/rejected": -1.371268630027771, + "logps/chosen": -0.29972949624061584, + "logps/rejected": -155.9679412841797, + "loss": 0.0293583944439888, + "memory(GiB)": 41.24, + "nll_loss": 0.027720170095562935, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22072438895702362, + "rewards/margins": 14.090465545654297, + "rewards/rejected": -13.869739532470703, + "step": 386, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.224864864864865, + "grad_norm": 0.32941335439682007, + "learning_rate": 3.333989146679458e-05, + "logits/chosen": -0.8679807782173157, + "logits/rejected": -1.1114749908447266, + "logps/chosen": -3.313974142074585, + "logps/rejected": -105.611083984375, + "loss": 0.19679458439350128, + "memory(GiB)": 41.24, + "nll_loss": 0.1846945434808731, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7463847398757935, + "rewards/margins": 10.00084400177002, + "rewards/rejected": -9.254459381103516, + "step": 387, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.2306306306306305, + "grad_norm": 1.0224090814590454, + "learning_rate": 3.286624018033389e-05, + "logits/chosen": -0.7229690551757812, + "logits/rejected": -1.231992244720459, + "logps/chosen": -2.896671772003174, + "logps/rejected": -146.92176818847656, + "loss": 0.15353035926818848, + "memory(GiB)": 41.24, + "nll_loss": 0.14388211071491241, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3378286361694336, + "rewards/margins": 13.106163024902344, + "rewards/rejected": -12.76833438873291, + "step": 388, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.236396396396396, + "grad_norm": 0.580308735370636, + "learning_rate": 3.2395315016627195e-05, + "logits/chosen": -0.7388817071914673, + "logits/rejected": -1.2255033254623413, + "logps/chosen": -1.828582525253296, + "logps/rejected": -164.5238494873047, + "loss": 0.08867667615413666, + "memory(GiB)": 41.24, + "nll_loss": 0.07473144680261612, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21864542365074158, + "rewards/margins": 14.580236434936523, + "rewards/rejected": -14.361591339111328, + "step": 389, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.2421621621621624, + "grad_norm": 0.6406400203704834, + "learning_rate": 3.1927135098688056e-05, + "logits/chosen": -0.7472262382507324, + "logits/rejected": -1.214005470275879, + "logps/chosen": -1.0447490215301514, + "logps/rejected": -148.38214111328125, + "loss": 0.08439233154058456, + "memory(GiB)": 41.24, + "nll_loss": 0.08170553296804428, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.30276942253112793, + "rewards/margins": 13.188764572143555, + "rewards/rejected": -12.885995864868164, + "step": 390, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.247927927927928, + "grad_norm": 0.847642183303833, + "learning_rate": 3.146171943805307e-05, + "logits/chosen": -0.7843738794326782, + "logits/rejected": -1.210669994354248, + "logps/chosen": -2.5757925510406494, + "logps/rejected": -166.5076904296875, + "loss": 0.22211028635501862, + "memory(GiB)": 41.24, + "nll_loss": 0.21541990339756012, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20775160193443298, + "rewards/margins": 14.521588325500488, + "rewards/rejected": -14.313838005065918, + "step": 391, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.2536936936936938, + "grad_norm": 1.2409991025924683, + "learning_rate": 3.0999086934009625e-05, + "logits/chosen": -0.7450385093688965, + "logits/rejected": -1.2391735315322876, + "logps/chosen": -1.3592970371246338, + "logps/rejected": -147.8185577392578, + "loss": 0.11896778643131256, + "memory(GiB)": 41.24, + "nll_loss": 0.11766871809959412, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3634198307991028, + "rewards/margins": 13.053730010986328, + "rewards/rejected": -12.690309524536133, + "step": 392, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.2594594594594595, + "grad_norm": 1.0117156505584717, + "learning_rate": 3.053925637282856e-05, + "logits/chosen": -0.8271034955978394, + "logits/rejected": -1.2085802555084229, + "logps/chosen": -0.8595653176307678, + "logps/rejected": -117.42967987060547, + "loss": 0.0890599861741066, + "memory(GiB)": 41.24, + "nll_loss": 0.08192116022109985, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32370027899742126, + "rewards/margins": 10.56024169921875, + "rewards/rejected": -10.236541748046875, + "step": 393, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.265225225225225, + "grad_norm": 0.702856183052063, + "learning_rate": 3.0082246427001347e-05, + "logits/chosen": -0.9588988423347473, + "logits/rejected": -1.2490991353988647, + "logps/chosen": -1.6205388307571411, + "logps/rejected": -128.76576232910156, + "loss": 0.14746464788913727, + "memory(GiB)": 41.24, + "nll_loss": 0.13545851409435272, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.29286202788352966, + "rewards/margins": 11.796010971069336, + "rewards/rejected": -11.503148078918457, + "step": 394, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.270990990990991, + "grad_norm": 0.977766215801239, + "learning_rate": 2.962807565448179e-05, + "logits/chosen": -0.8526955842971802, + "logits/rejected": -1.2339715957641602, + "logps/chosen": -0.8969095945358276, + "logps/rejected": -130.5006103515625, + "loss": 0.07543729990720749, + "memory(GiB)": 41.24, + "nll_loss": 0.07074794173240662, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3233695328235626, + "rewards/margins": 11.610946655273438, + "rewards/rejected": -11.287576675415039, + "step": 395, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.2767567567567566, + "grad_norm": 0.37084048986434937, + "learning_rate": 2.9176762497932375e-05, + "logits/chosen": -0.7118812799453735, + "logits/rejected": -1.1689503192901611, + "logps/chosen": -2.4797844886779785, + "logps/rejected": -150.7263946533203, + "loss": 0.1401955485343933, + "memory(GiB)": 41.24, + "nll_loss": 0.13063007593154907, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.518430769443512, + "rewards/margins": 13.590339660644531, + "rewards/rejected": -13.071908950805664, + "step": 396, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.2825225225225223, + "grad_norm": 0.6788544654846191, + "learning_rate": 2.8728325283975553e-05, + "logits/chosen": -0.8885631561279297, + "logits/rejected": -1.2710957527160645, + "logps/chosen": -0.9696691632270813, + "logps/rejected": -132.19674682617188, + "loss": 0.05637747049331665, + "memory(GiB)": 41.24, + "nll_loss": 0.053990356624126434, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.682148277759552, + "rewards/margins": 12.16622543334961, + "rewards/rejected": -11.484075546264648, + "step": 397, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.2882882882882885, + "grad_norm": 0.6900463700294495, + "learning_rate": 2.8282782222449267e-05, + "logits/chosen": -0.7823069095611572, + "logits/rejected": -1.2541143894195557, + "logps/chosen": -1.974277138710022, + "logps/rejected": -139.66693115234375, + "loss": 0.11486522108316422, + "memory(GiB)": 41.24, + "nll_loss": 0.11099116504192352, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.28144386410713196, + "rewards/margins": 12.4359769821167, + "rewards/rejected": -12.154533386230469, + "step": 398, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.294054054054054, + "grad_norm": 0.5850529670715332, + "learning_rate": 2.7840151405667837e-05, + "logits/chosen": -0.7849150896072388, + "logits/rejected": -1.1549279689788818, + "logps/chosen": -1.7644596099853516, + "logps/rejected": -127.66755676269531, + "loss": 0.1364857256412506, + "memory(GiB)": 41.24, + "nll_loss": 0.08438676595687866, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7024877071380615, + "rewards/margins": 12.10548210144043, + "rewards/rejected": -11.402995109558105, + "step": 399, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.29981981981982, + "grad_norm": 0.7643297910690308, + "learning_rate": 2.7400450807686938e-05, + "logits/chosen": -0.8377431035041809, + "logits/rejected": -1.2140753269195557, + "logps/chosen": -2.0960607528686523, + "logps/rejected": -125.79196166992188, + "loss": 0.24046017229557037, + "memory(GiB)": 41.24, + "nll_loss": 0.2326795905828476, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1421297788619995, + "rewards/margins": 11.214141845703125, + "rewards/rejected": -11.072011947631836, + "step": 400, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.3055855855855856, + "grad_norm": 1.6450413465499878, + "learning_rate": 2.6963698283573958e-05, + "logits/chosen": -0.7968766689300537, + "logits/rejected": -1.177986979484558, + "logps/chosen": -2.587327241897583, + "logps/rejected": -113.06391906738281, + "loss": 0.17521585524082184, + "memory(GiB)": 41.24, + "nll_loss": 0.16903801262378693, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4117201864719391, + "rewards/margins": 10.178709030151367, + "rewards/rejected": -9.766987800598145, + "step": 401, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.3113513513513513, + "grad_norm": 1.0271110534667969, + "learning_rate": 2.6529911568682876e-05, + "logits/chosen": -0.843456506729126, + "logits/rejected": -1.1689084768295288, + "logps/chosen": -1.888105034828186, + "logps/rejected": -122.53902435302734, + "loss": 0.11974243819713593, + "memory(GiB)": 41.24, + "nll_loss": 0.1172693520784378, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5889266133308411, + "rewards/margins": 11.48852252960205, + "rewards/rejected": -10.8995943069458, + "step": 402, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.317117117117117, + "grad_norm": 0.7545397877693176, + "learning_rate": 2.6099108277934103e-05, + "logits/chosen": -0.9295822978019714, + "logits/rejected": -1.2397063970565796, + "logps/chosen": -0.5876428484916687, + "logps/rejected": -123.83692932128906, + "loss": 0.07338391244411469, + "memory(GiB)": 41.24, + "nll_loss": 0.0500425323843956, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5376147627830505, + "rewards/margins": 11.60782241821289, + "rewards/rejected": -11.070207595825195, + "step": 403, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.3228828828828827, + "grad_norm": 0.564996063709259, + "learning_rate": 2.5671305905099075e-05, + "logits/chosen": -0.8100417852401733, + "logits/rejected": -1.2899115085601807, + "logps/chosen": -1.0249552726745605, + "logps/rejected": -145.04527282714844, + "loss": 0.06945691257715225, + "memory(GiB)": 41.24, + "nll_loss": 0.06294979900121689, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4287530481815338, + "rewards/margins": 13.313419342041016, + "rewards/rejected": -12.884666442871094, + "step": 404, + "train_speed(iter/s)": 0.010719 + }, + { + "epoch": 2.328648648648649, + "grad_norm": 0.5819610953330994, + "learning_rate": 2.5246521822090064e-05, + "logits/chosen": -0.8548004627227783, + "logits/rejected": -1.254213809967041, + "logps/chosen": -1.3816176652908325, + "logps/rejected": -128.0045623779297, + "loss": 0.10609627515077591, + "memory(GiB)": 41.24, + "nll_loss": 0.08565448224544525, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34351933002471924, + "rewards/margins": 11.546485900878906, + "rewards/rejected": -11.20296573638916, + "step": 405, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.3344144144144146, + "grad_norm": 0.5590229630470276, + "learning_rate": 2.4824773278254544e-05, + "logits/chosen": -0.8491780757904053, + "logits/rejected": -1.343767523765564, + "logps/chosen": -0.6148195266723633, + "logps/rejected": -144.7222442626953, + "loss": 0.08045926690101624, + "memory(GiB)": 41.24, + "nll_loss": 0.0773516446352005, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16773077845573425, + "rewards/margins": 12.882085800170898, + "rewards/rejected": -12.71435546875, + "step": 406, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.3401801801801803, + "grad_norm": 0.6553923487663269, + "learning_rate": 2.4406077399674963e-05, + "logits/chosen": -0.8346514701843262, + "logits/rejected": -1.2444106340408325, + "logps/chosen": -1.604792833328247, + "logps/rejected": -134.56263732910156, + "loss": 0.11761439591646194, + "memory(GiB)": 41.24, + "nll_loss": 0.11513374000787735, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2989213764667511, + "rewards/margins": 12.23564338684082, + "rewards/rejected": -11.936722755432129, + "step": 407, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.345945945945946, + "grad_norm": 0.7419153451919556, + "learning_rate": 2.3990451188473073e-05, + "logits/chosen": -0.7969120740890503, + "logits/rejected": -1.3061842918395996, + "logps/chosen": -0.3484644293785095, + "logps/rejected": -142.5753631591797, + "loss": 0.053259484469890594, + "memory(GiB)": 41.24, + "nll_loss": 0.035422272980213165, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17469555139541626, + "rewards/margins": 12.486812591552734, + "rewards/rejected": -12.312116622924805, + "step": 408, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.3517117117117117, + "grad_norm": 0.42467495799064636, + "learning_rate": 2.35779115221197e-05, + "logits/chosen": -0.8599014282226562, + "logits/rejected": -1.1873267889022827, + "logps/chosen": -0.8807926177978516, + "logps/rejected": -131.24581909179688, + "loss": 0.0740325078368187, + "memory(GiB)": 41.24, + "nll_loss": 0.0588376484811306, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.29491689801216125, + "rewards/margins": 11.817724227905273, + "rewards/rejected": -11.522806167602539, + "step": 409, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.3574774774774774, + "grad_norm": 0.8155800104141235, + "learning_rate": 2.3168475152749346e-05, + "logits/chosen": -0.8553993105888367, + "logits/rejected": -1.3119568824768066, + "logps/chosen": -0.9577528834342957, + "logps/rejected": -145.17288208007812, + "loss": 0.11313124001026154, + "memory(GiB)": 41.24, + "nll_loss": 0.11024200916290283, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.288902223110199, + "rewards/margins": 13.217832565307617, + "rewards/rejected": -12.928930282592773, + "step": 410, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.363243243243243, + "grad_norm": 0.7672101259231567, + "learning_rate": 2.276215870647983e-05, + "logits/chosen": -0.7228320837020874, + "logits/rejected": -1.2324626445770264, + "logps/chosen": -0.5273553133010864, + "logps/rejected": -129.128662109375, + "loss": 0.05981205031275749, + "memory(GiB)": 41.24, + "nll_loss": 0.055507220327854156, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3216668367385864, + "rewards/margins": 11.372444152832031, + "rewards/rejected": -11.050777435302734, + "step": 411, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.369009009009009, + "grad_norm": 0.5394435524940491, + "learning_rate": 2.235897868273723e-05, + "logits/chosen": -0.8678345084190369, + "logits/rejected": -1.130922794342041, + "logps/chosen": -2.958963394165039, + "logps/rejected": -117.44721221923828, + "loss": 0.189490407705307, + "memory(GiB)": 41.24, + "nll_loss": 0.17878007888793945, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.36166027188301086, + "rewards/margins": 10.397054672241211, + "rewards/rejected": -10.035394668579102, + "step": 412, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.374774774774775, + "grad_norm": 2.8132057189941406, + "learning_rate": 2.1958951453585964e-05, + "logits/chosen": -0.8280112743377686, + "logits/rejected": -1.213489294052124, + "logps/chosen": -1.552673101425171, + "logps/rejected": -117.25660705566406, + "loss": 0.1538255214691162, + "memory(GiB)": 41.24, + "nll_loss": 0.12946051359176636, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.46080461144447327, + "rewards/margins": 10.577008247375488, + "rewards/rejected": -10.116203308105469, + "step": 413, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.3805405405405407, + "grad_norm": 1.1900241374969482, + "learning_rate": 2.1562093263063777e-05, + "logits/chosen": -0.8969691395759583, + "logits/rejected": -1.2387640476226807, + "logps/chosen": -3.069331407546997, + "logps/rejected": -134.249755859375, + "loss": 0.1963363140821457, + "memory(GiB)": 41.24, + "nll_loss": 0.19451972842216492, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.46095818281173706, + "rewards/margins": 12.321144104003906, + "rewards/rejected": -11.860186576843262, + "step": 414, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.3863063063063064, + "grad_norm": 1.5336196422576904, + "learning_rate": 2.116842022652228e-05, + "logits/chosen": -0.8248907327651978, + "logits/rejected": -1.1561063528060913, + "logps/chosen": -2.3276851177215576, + "logps/rejected": -132.52685546875, + "loss": 0.11701621115207672, + "memory(GiB)": 41.24, + "nll_loss": 0.10921375453472137, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4955483675003052, + "rewards/margins": 12.521132469177246, + "rewards/rejected": -12.025585174560547, + "step": 415, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.392072072072072, + "grad_norm": 0.7101905941963196, + "learning_rate": 2.0777948329972497e-05, + "logits/chosen": -0.7113717198371887, + "logits/rejected": -1.2030726671218872, + "logps/chosen": -0.581327497959137, + "logps/rejected": -148.84197998046875, + "loss": 0.07604608684778214, + "memory(GiB)": 41.24, + "nll_loss": 0.07405771315097809, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6247385740280151, + "rewards/margins": 13.593480110168457, + "rewards/rejected": -12.968741416931152, + "step": 416, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.3978378378378378, + "grad_norm": 0.5968648791313171, + "learning_rate": 2.0390693429435627e-05, + "logits/chosen": -0.8398076295852661, + "logits/rejected": -1.2533546686172485, + "logps/chosen": -2.379636526107788, + "logps/rejected": -136.64157104492188, + "loss": 0.10497608780860901, + "memory(GiB)": 41.24, + "nll_loss": 0.10299289226531982, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3265537619590759, + "rewards/margins": 12.218026161193848, + "rewards/rejected": -11.891471862792969, + "step": 417, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.4036036036036035, + "grad_norm": 0.7634880542755127, + "learning_rate": 2.0006671250299337e-05, + "logits/chosen": -0.8096540570259094, + "logits/rejected": -1.2810704708099365, + "logps/chosen": -0.9330279231071472, + "logps/rejected": -132.92745971679688, + "loss": 0.0640283152461052, + "memory(GiB)": 41.24, + "nll_loss": 0.06352468580007553, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.45129770040512085, + "rewards/margins": 11.928510665893555, + "rewards/rejected": -11.477213859558105, + "step": 418, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.409369369369369, + "grad_norm": 0.732837975025177, + "learning_rate": 1.9625897386679038e-05, + "logits/chosen": -0.7635003328323364, + "logits/rejected": -1.2484025955200195, + "logps/chosen": -1.4449158906936646, + "logps/rejected": -153.7145233154297, + "loss": 0.23150083422660828, + "memory(GiB)": 41.24, + "nll_loss": 0.1963690221309662, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.34227463603019714, + "rewards/margins": 13.607999801635742, + "rewards/rejected": -13.265726089477539, + "step": 419, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.4151351351351353, + "grad_norm": 1.821128249168396, + "learning_rate": 1.924838730078474e-05, + "logits/chosen": -0.7961917519569397, + "logits/rejected": -1.1534490585327148, + "logps/chosen": -2.9788126945495605, + "logps/rejected": -116.79434967041016, + "loss": 0.29703086614608765, + "memory(GiB)": 41.24, + "nll_loss": 0.29087352752685547, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5089370608329773, + "rewards/margins": 10.513839721679688, + "rewards/rejected": -10.004903793334961, + "step": 420, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.420900900900901, + "grad_norm": 18.245189666748047, + "learning_rate": 1.887415632229318e-05, + "logits/chosen": -0.8124462366104126, + "logits/rejected": -1.241105318069458, + "logps/chosen": -1.428385853767395, + "logps/rejected": -122.31611633300781, + "loss": 0.1208016499876976, + "memory(GiB)": 41.24, + "nll_loss": 0.11244743317365646, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.38437214493751526, + "rewards/margins": 10.756612777709961, + "rewards/rejected": -10.372241020202637, + "step": 421, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.4266666666666667, + "grad_norm": 0.7744676470756531, + "learning_rate": 1.850321964772528e-05, + "logits/chosen": -0.7525817155838013, + "logits/rejected": -1.1792981624603271, + "logps/chosen": -1.8876060247421265, + "logps/rejected": -139.65049743652344, + "loss": 0.13217130303382874, + "memory(GiB)": 41.24, + "nll_loss": 0.12513425946235657, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.396883487701416, + "rewards/margins": 12.279129028320312, + "rewards/rejected": -11.882246017456055, + "step": 422, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.4324324324324325, + "grad_norm": 0.89878249168396, + "learning_rate": 1.8135592339829098e-05, + "logits/chosen": -0.7277660369873047, + "logits/rejected": -1.199865698814392, + "logps/chosen": -0.8733133673667908, + "logps/rejected": -133.43618774414062, + "loss": 0.07002358138561249, + "memory(GiB)": 41.24, + "nll_loss": 0.06312312930822372, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3732052445411682, + "rewards/margins": 12.146612167358398, + "rewards/rejected": -11.773406982421875, + "step": 423, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.438198198198198, + "grad_norm": 0.5303387641906738, + "learning_rate": 1.7771289326968098e-05, + "logits/chosen": -0.7044744491577148, + "logits/rejected": -1.175467610359192, + "logps/chosen": -1.2831920385360718, + "logps/rejected": -143.45713806152344, + "loss": 0.11005179584026337, + "memory(GiB)": 41.24, + "nll_loss": 0.09954062849283218, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20419657230377197, + "rewards/margins": 12.66477108001709, + "rewards/rejected": -12.46057415008545, + "step": 424, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.443963963963964, + "grad_norm": 0.6761723160743713, + "learning_rate": 1.7410325402515003e-05, + "logits/chosen": -0.7268787622451782, + "logits/rejected": -1.1966400146484375, + "logps/chosen": -1.524604082107544, + "logps/rejected": -144.5257568359375, + "loss": 0.10763537883758545, + "memory(GiB)": 41.24, + "nll_loss": 0.1054452508687973, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7338725328445435, + "rewards/margins": 13.085524559020996, + "rewards/rejected": -12.351653099060059, + "step": 425, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.4497297297297296, + "grad_norm": 0.6517868638038635, + "learning_rate": 1.7052715224251147e-05, + "logits/chosen": -0.9204325675964355, + "logits/rejected": -1.1545764207839966, + "logps/chosen": -1.6804869174957275, + "logps/rejected": -119.65060424804688, + "loss": 0.09586860239505768, + "memory(GiB)": 41.24, + "nll_loss": 0.08874297887086868, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6586800813674927, + "rewards/margins": 11.184593200683594, + "rewards/rejected": -10.525914192199707, + "step": 426, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.4554954954954953, + "grad_norm": 0.5355521440505981, + "learning_rate": 1.669847331377109e-05, + "logits/chosen": -0.702139139175415, + "logits/rejected": -1.2590887546539307, + "logps/chosen": -1.4097001552581787, + "logps/rejected": -172.09359741210938, + "loss": 0.13867531716823578, + "memory(GiB)": 41.24, + "nll_loss": 0.09702995419502258, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.37929418683052063, + "rewards/margins": 14.985157012939453, + "rewards/rejected": -14.605862617492676, + "step": 427, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.4612612612612614, + "grad_norm": 0.6977973580360413, + "learning_rate": 1.6347614055893055e-05, + "logits/chosen": -0.8643964529037476, + "logits/rejected": -1.2080752849578857, + "logps/chosen": -2.0021631717681885, + "logps/rejected": -112.87459564208984, + "loss": 0.10970260947942734, + "memory(GiB)": 41.24, + "nll_loss": 0.10636736452579498, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.49944260716438293, + "rewards/margins": 10.13442325592041, + "rewards/rejected": -9.634980201721191, + "step": 428, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.467027027027027, + "grad_norm": 1.0196927785873413, + "learning_rate": 1.6000151698074816e-05, + "logits/chosen": -0.7368990182876587, + "logits/rejected": -1.236674427986145, + "logps/chosen": -0.9173886179924011, + "logps/rejected": -139.41201782226562, + "loss": 0.08401184529066086, + "memory(GiB)": 41.24, + "nll_loss": 0.08191582560539246, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.30264952778816223, + "rewards/margins": 12.492189407348633, + "rewards/rejected": -12.189539909362793, + "step": 429, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.472792792792793, + "grad_norm": 0.6740352511405945, + "learning_rate": 1.5656100349835057e-05, + "logits/chosen": -0.6987006068229675, + "logits/rejected": -1.150787591934204, + "logps/chosen": -1.9186586141586304, + "logps/rejected": -117.69166564941406, + "loss": 0.27147069573402405, + "memory(GiB)": 41.24, + "nll_loss": 0.1855418086051941, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2978801727294922, + "rewards/margins": 10.272615432739258, + "rewards/rejected": -9.974735260009766, + "step": 430, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.4785585585585586, + "grad_norm": 1.4962725639343262, + "learning_rate": 1.531547398218053e-05, + "logits/chosen": -0.6737241744995117, + "logits/rejected": -1.2129101753234863, + "logps/chosen": -2.0417466163635254, + "logps/rejected": -162.4108123779297, + "loss": 0.1547425389289856, + "memory(GiB)": 41.24, + "nll_loss": 0.149099200963974, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.36950764060020447, + "rewards/margins": 14.570940017700195, + "rewards/rejected": -14.201431274414062, + "step": 431, + "train_speed(iter/s)": 0.010722 + }, + { + "epoch": 2.4843243243243243, + "grad_norm": 0.9512774348258972, + "learning_rate": 1.4978286427038601e-05, + "logits/chosen": -0.8025528788566589, + "logits/rejected": -1.2414032220840454, + "logps/chosen": -0.7892335057258606, + "logps/rejected": -127.80113220214844, + "loss": 0.08720188587903976, + "memory(GiB)": 41.24, + "nll_loss": 0.08405375480651855, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3307220935821533, + "rewards/margins": 11.417773246765137, + "rewards/rejected": -11.087050437927246, + "step": 432, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.49009009009009, + "grad_norm": 0.9329023361206055, + "learning_rate": 1.4644551376695636e-05, + "logits/chosen": -0.8390129804611206, + "logits/rejected": -1.2667245864868164, + "logps/chosen": -0.6291428804397583, + "logps/rejected": -139.28102111816406, + "loss": 0.04609063267707825, + "memory(GiB)": 41.24, + "nll_loss": 0.04203294590115547, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.42254215478897095, + "rewards/margins": 12.474265098571777, + "rewards/rejected": -12.051721572875977, + "step": 433, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.4958558558558557, + "grad_norm": 0.33022162318229675, + "learning_rate": 1.4314282383241096e-05, + "logits/chosen": -0.9568659067153931, + "logits/rejected": -1.2154091596603394, + "logps/chosen": -1.4253203868865967, + "logps/rejected": -107.42050170898438, + "loss": 0.1329001784324646, + "memory(GiB)": 41.24, + "nll_loss": 0.1263602375984192, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3609982132911682, + "rewards/margins": 9.884754180908203, + "rewards/rejected": -9.52375602722168, + "step": 434, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.501621621621622, + "grad_norm": 0.6514727473258972, + "learning_rate": 1.3987492858016994e-05, + "logits/chosen": -0.7563194036483765, + "logits/rejected": -1.1865994930267334, + "logps/chosen": -1.2849355936050415, + "logps/rejected": -124.59906768798828, + "loss": 0.08598919212818146, + "memory(GiB)": 41.24, + "nll_loss": 0.070497065782547, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4762778878211975, + "rewards/margins": 11.114522933959961, + "rewards/rejected": -10.63824462890625, + "step": 435, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.5073873873873875, + "grad_norm": 0.5046420693397522, + "learning_rate": 1.3664196071073521e-05, + "logits/chosen": -0.7833178043365479, + "logits/rejected": -1.1460063457489014, + "logps/chosen": -1.5084190368652344, + "logps/rejected": -127.19114685058594, + "loss": 0.09911272674798965, + "memory(GiB)": 41.24, + "nll_loss": 0.09469584375619888, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.543127179145813, + "rewards/margins": 11.694902420043945, + "rewards/rejected": -11.151777267456055, + "step": 436, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.5131531531531532, + "grad_norm": 0.9791157841682434, + "learning_rate": 1.334440515063009e-05, + "logits/chosen": -0.7145990133285522, + "logits/rejected": -1.2396931648254395, + "logps/chosen": -1.3195858001708984, + "logps/rejected": -144.9882049560547, + "loss": 0.14285551011562347, + "memory(GiB)": 41.24, + "nll_loss": 0.13504329323768616, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24640637636184692, + "rewards/margins": 12.515280723571777, + "rewards/rejected": -12.26887321472168, + "step": 437, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.518918918918919, + "grad_norm": 0.7919338345527649, + "learning_rate": 1.3028133082542172e-05, + "logits/chosen": -0.728066623210907, + "logits/rejected": -1.209718942642212, + "logps/chosen": -1.6923828125, + "logps/rejected": -140.62538146972656, + "loss": 0.11857596039772034, + "memory(GiB)": 41.24, + "nll_loss": 0.11159586906433105, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.33942872285842896, + "rewards/margins": 12.54323959350586, + "rewards/rejected": -12.203811645507812, + "step": 438, + "train_speed(iter/s)": 0.010723 + }, + { + "epoch": 2.5246846846846847, + "grad_norm": 0.8548752069473267, + "learning_rate": 1.2715392709774099e-05, + "logits/chosen": -0.8878112435340881, + "logits/rejected": -1.135077714920044, + "logps/chosen": -1.7542343139648438, + "logps/rejected": -110.59034729003906, + "loss": 0.1734868437051773, + "memory(GiB)": 41.24, + "nll_loss": 0.14896531403064728, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4700812101364136, + "rewards/margins": 10.092097282409668, + "rewards/rejected": -9.622017860412598, + "step": 439, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5304504504504504, + "grad_norm": 0.5713933110237122, + "learning_rate": 1.2406196731877462e-05, + "logits/chosen": -0.7658921480178833, + "logits/rejected": -1.203586459159851, + "logps/chosen": -1.3810665607452393, + "logps/rejected": -126.4085464477539, + "loss": 0.08227439224720001, + "memory(GiB)": 41.24, + "nll_loss": 0.07479289174079895, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.421691358089447, + "rewards/margins": 11.285358428955078, + "rewards/rejected": -10.863666534423828, + "step": 440, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.536216216216216, + "grad_norm": 0.7913537621498108, + "learning_rate": 1.2100557704475401e-05, + "logits/chosen": -0.4924860894680023, + "logits/rejected": -1.1296981573104858, + "logps/chosen": -1.3359569311141968, + "logps/rejected": -132.1542205810547, + "loss": 0.10920162498950958, + "memory(GiB)": 41.24, + "nll_loss": 0.09422454237937927, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3951561152935028, + "rewards/margins": 11.186981201171875, + "rewards/rejected": -10.791825294494629, + "step": 441, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5419819819819818, + "grad_norm": 0.64546138048172, + "learning_rate": 1.1798488038752853e-05, + "logits/chosen": -0.7858179211616516, + "logits/rejected": -1.1689412593841553, + "logps/chosen": -1.757339596748352, + "logps/rejected": -134.68630981445312, + "loss": 0.1118108257651329, + "memory(GiB)": 41.24, + "nll_loss": 0.10429531335830688, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.42179539799690247, + "rewards/margins": 11.935916900634766, + "rewards/rejected": -11.514120101928711, + "step": 442, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5477477477477475, + "grad_norm": 0.5738776326179504, + "learning_rate": 1.1500000000952516e-05, + "logits/chosen": -0.7904771566390991, + "logits/rejected": -1.305051326751709, + "logps/chosen": -0.38108402490615845, + "logps/rejected": -135.01937866210938, + "loss": 0.047188352793455124, + "memory(GiB)": 41.24, + "nll_loss": 0.044908784329891205, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2293405830860138, + "rewards/margins": 12.095958709716797, + "rewards/rejected": -11.866617202758789, + "step": 443, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5535135135135136, + "grad_norm": 0.3696437478065491, + "learning_rate": 1.1205105711876651e-05, + "logits/chosen": -0.6969178915023804, + "logits/rejected": -1.2377140522003174, + "logps/chosen": -0.9388285875320435, + "logps/rejected": -148.32156372070312, + "loss": 0.08691015839576721, + "memory(GiB)": 41.24, + "nll_loss": 0.06484532356262207, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.311213880777359, + "rewards/margins": 12.803532600402832, + "rewards/rejected": -12.492319107055664, + "step": 444, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5592792792792793, + "grad_norm": 0.6557607054710388, + "learning_rate": 1.0913817146395088e-05, + "logits/chosen": -0.6606895923614502, + "logits/rejected": -1.2088009119033813, + "logps/chosen": -1.2118579149246216, + "logps/rejected": -157.25430297851562, + "loss": 0.13226519525051117, + "memory(GiB)": 41.24, + "nll_loss": 0.12357373535633087, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2107386589050293, + "rewards/margins": 13.488852500915527, + "rewards/rejected": -13.278114318847656, + "step": 445, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.565045045045045, + "grad_norm": 1.1071375608444214, + "learning_rate": 1.0626146132958759e-05, + "logits/chosen": -0.8382202386856079, + "logits/rejected": -1.1532580852508545, + "logps/chosen": -1.4709382057189941, + "logps/rejected": -111.8001708984375, + "loss": 0.09750804305076599, + "memory(GiB)": 41.24, + "nll_loss": 0.07481691986322403, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5700428485870361, + "rewards/margins": 10.36096477508545, + "rewards/rejected": -9.790922164916992, + "step": 446, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5708108108108108, + "grad_norm": 0.5680884122848511, + "learning_rate": 1.034210435311952e-05, + "logits/chosen": -0.7516598701477051, + "logits/rejected": -1.2296390533447266, + "logps/chosen": -1.1601189374923706, + "logps/rejected": -134.99639892578125, + "loss": 0.14666548371315002, + "memory(GiB)": 41.24, + "nll_loss": 0.13436883687973022, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16603690385818481, + "rewards/margins": 12.123270988464355, + "rewards/rejected": -11.957234382629395, + "step": 447, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5765765765765765, + "grad_norm": 1.6710106134414673, + "learning_rate": 1.0061703341055706e-05, + "logits/chosen": -0.7993563413619995, + "logits/rejected": -1.1708520650863647, + "logps/chosen": -1.499343991279602, + "logps/rejected": -116.2354965209961, + "loss": 0.12997783720493317, + "memory(GiB)": 41.24, + "nll_loss": 0.11543214321136475, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5618722438812256, + "rewards/margins": 10.406671524047852, + "rewards/rejected": -9.844799041748047, + "step": 448, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5823423423423426, + "grad_norm": 1.043412208557129, + "learning_rate": 9.784954483103803e-06, + "logits/chosen": -0.7738111019134521, + "logits/rejected": -1.2526034116744995, + "logps/chosen": -0.8989908695220947, + "logps/rejected": -129.46839904785156, + "loss": 0.09322156012058258, + "memory(GiB)": 41.24, + "nll_loss": 0.08673778921365738, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2142462432384491, + "rewards/margins": 11.282845497131348, + "rewards/rejected": -11.068598747253418, + "step": 449, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.5881081081081083, + "grad_norm": 0.5241620540618896, + "learning_rate": 9.511869017296116e-06, + "logits/chosen": -0.8610420227050781, + "logits/rejected": -1.290226936340332, + "logps/chosen": -1.1238694190979004, + "logps/rejected": -140.24301147460938, + "loss": 0.08800040930509567, + "memory(GiB)": 41.24, + "nll_loss": 0.08061891794204712, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1996644139289856, + "rewards/margins": 12.624069213867188, + "rewards/rejected": -12.42440414428711, + "step": 450, + "train_speed(iter/s)": 0.010724 + }, + { + "epoch": 2.593873873873874, + "grad_norm": 0.8182636499404907, + "learning_rate": 9.242458032904311e-06, + "logits/chosen": -0.6504684090614319, + "logits/rejected": -1.1887084245681763, + "logps/chosen": -1.885643720626831, + "logps/rejected": -139.11285400390625, + "loss": 0.11732271313667297, + "memory(GiB)": 41.24, + "nll_loss": 0.1091528832912445, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.49910297989845276, + "rewards/margins": 11.791345596313477, + "rewards/rejected": -11.292243003845215, + "step": 451, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.5996396396396397, + "grad_norm": 0.6420908570289612, + "learning_rate": 8.976732469989157e-06, + "logits/chosen": -0.8639041185379028, + "logits/rejected": -1.136501669883728, + "logps/chosen": -2.2670693397521973, + "logps/rejected": -110.9341049194336, + "loss": 0.11834051460027695, + "memory(GiB)": 41.24, + "nll_loss": 0.11381684988737106, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4113590717315674, + "rewards/margins": 10.21336555480957, + "rewards/rejected": -9.802005767822266, + "step": 452, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.6054054054054054, + "grad_norm": 0.6626132130622864, + "learning_rate": 8.714703118956402e-06, + "logits/chosen": -0.8029040098190308, + "logits/rejected": -1.084507703781128, + "logps/chosen": -2.472752809524536, + "logps/rejected": -112.6861343383789, + "loss": 0.14413487911224365, + "memory(GiB)": 41.24, + "nll_loss": 0.13469503819942474, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5375735759735107, + "rewards/margins": 9.903990745544434, + "rewards/rejected": -9.366416931152344, + "step": 453, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.611171171171171, + "grad_norm": 2.2931597232818604, + "learning_rate": 8.456380620118365e-06, + "logits/chosen": -0.8448130488395691, + "logits/rejected": -1.2921265363693237, + "logps/chosen": -1.6435465812683105, + "logps/rejected": -140.62783813476562, + "loss": 0.16373160481452942, + "memory(GiB)": 41.24, + "nll_loss": 0.15887896716594696, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19452501833438873, + "rewards/margins": 12.709239959716797, + "rewards/rejected": -12.514715194702148, + "step": 454, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.616936936936937, + "grad_norm": 0.5038320422172546, + "learning_rate": 8.201775463262107e-06, + "logits/chosen": -0.6445741057395935, + "logits/rejected": -1.1499004364013672, + "logps/chosen": -0.7528181672096252, + "logps/rejected": -144.4021453857422, + "loss": 0.07080763578414917, + "memory(GiB)": 41.24, + "nll_loss": 0.04995018243789673, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34368783235549927, + "rewards/margins": 12.341341972351074, + "rewards/rejected": -11.99765396118164, + "step": 455, + "train_speed(iter/s)": 0.01072 + }, + { + "epoch": 2.6227027027027026, + "grad_norm": 0.47270169854164124, + "learning_rate": 7.950897987223304e-06, + "logits/chosen": -0.8239877820014954, + "logits/rejected": -1.1374708414077759, + "logps/chosen": -2.5196967124938965, + "logps/rejected": -115.45918273925781, + "loss": 0.1638275384902954, + "memory(GiB)": 41.24, + "nll_loss": 0.16180458664894104, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5195181369781494, + "rewards/margins": 10.506765365600586, + "rewards/rejected": -9.987248420715332, + "step": 456, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.6284684684684683, + "grad_norm": 0.863614022731781, + "learning_rate": 7.703758379466441e-06, + "logits/chosen": -0.752800464630127, + "logits/rejected": -1.1401550769805908, + "logps/chosen": -1.9722648859024048, + "logps/rejected": -114.79458618164062, + "loss": 0.1980990469455719, + "memory(GiB)": 41.24, + "nll_loss": 0.17658959329128265, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3625020682811737, + "rewards/margins": 10.017668724060059, + "rewards/rejected": -9.655166625976562, + "step": 457, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.634234234234234, + "grad_norm": 22.391117095947266, + "learning_rate": 7.460366675671215e-06, + "logits/chosen": -0.7305208444595337, + "logits/rejected": -1.1546024084091187, + "logps/chosen": -0.7069193720817566, + "logps/rejected": -114.72402954101562, + "loss": 0.10258577764034271, + "memory(GiB)": 41.24, + "nll_loss": 0.08164913207292557, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21952958405017853, + "rewards/margins": 9.828356742858887, + "rewards/rejected": -9.608826637268066, + "step": 458, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.64, + "grad_norm": 0.6944244503974915, + "learning_rate": 7.220732759324911e-06, + "logits/chosen": -0.6636066436767578, + "logits/rejected": -1.1836016178131104, + "logps/chosen": -1.4806097745895386, + "logps/rejected": -140.08389282226562, + "loss": 0.06510600447654724, + "memory(GiB)": 41.24, + "nll_loss": 0.06403350085020065, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3904286026954651, + "rewards/margins": 12.330760955810547, + "rewards/rejected": -11.940332412719727, + "step": 459, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.645765765765766, + "grad_norm": 0.6314659714698792, + "learning_rate": 6.984866361321063e-06, + "logits/chosen": -0.834367036819458, + "logits/rejected": -1.1780287027359009, + "logps/chosen": -1.9520055055618286, + "logps/rejected": -116.7628173828125, + "loss": 0.12525717914104462, + "memory(GiB)": 41.24, + "nll_loss": 0.1105702668428421, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5963611602783203, + "rewards/margins": 10.775897979736328, + "rewards/rejected": -10.179536819458008, + "step": 460, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.6515315315315315, + "grad_norm": 0.7644787430763245, + "learning_rate": 6.75277705956443e-06, + "logits/chosen": -0.8445948362350464, + "logits/rejected": -1.1863865852355957, + "logps/chosen": -1.3802237510681152, + "logps/rejected": -116.82594299316406, + "loss": 0.09831662476062775, + "memory(GiB)": 41.24, + "nll_loss": 0.08528903871774673, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.311068058013916, + "rewards/margins": 10.458280563354492, + "rewards/rejected": -10.147211074829102, + "step": 461, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.6572972972972972, + "grad_norm": 0.7514888048171997, + "learning_rate": 6.524474278581905e-06, + "logits/chosen": -0.7422783970832825, + "logits/rejected": -1.1011567115783691, + "logps/chosen": -2.3441312313079834, + "logps/rejected": -124.40501403808594, + "loss": 0.14947795867919922, + "memory(GiB)": 41.24, + "nll_loss": 0.1097560003399849, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.41284674406051636, + "rewards/margins": 10.943521499633789, + "rewards/rejected": -10.530674934387207, + "step": 462, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.663063063063063, + "grad_norm": 0.6188684105873108, + "learning_rate": 6.299967289139896e-06, + "logits/chosen": -0.7526286244392395, + "logits/rejected": -1.2433913946151733, + "logps/chosen": -1.184843897819519, + "logps/rejected": -135.50680541992188, + "loss": 0.1157502830028534, + "memory(GiB)": 41.24, + "nll_loss": 0.10655991733074188, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4602788984775543, + "rewards/margins": 12.347315788269043, + "rewards/rejected": -11.88703727722168, + "step": 463, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.668828828828829, + "grad_norm": 0.9016402363777161, + "learning_rate": 6.079265207867901e-06, + "logits/chosen": -0.8139784336090088, + "logits/rejected": -1.1881388425827026, + "logps/chosen": -1.122667670249939, + "logps/rejected": -101.08941650390625, + "loss": 0.14926381409168243, + "memory(GiB)": 41.24, + "nll_loss": 0.12030842900276184, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26348748803138733, + "rewards/margins": 8.991544723510742, + "rewards/rejected": -8.728057861328125, + "step": 464, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.674594594594595, + "grad_norm": 1.4042491912841797, + "learning_rate": 5.862376996888175e-06, + "logits/chosen": -0.7062532305717468, + "logits/rejected": -1.1928894519805908, + "logps/chosen": -1.775325059890747, + "logps/rejected": -149.07627868652344, + "loss": 0.13146454095840454, + "memory(GiB)": 41.24, + "nll_loss": 0.12969717383384705, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5256021022796631, + "rewards/margins": 12.805788040161133, + "rewards/rejected": -12.280186653137207, + "step": 465, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.6803603603603605, + "grad_norm": 0.7757210731506348, + "learning_rate": 5.6493114634519455e-06, + "logits/chosen": -0.7739042043685913, + "logits/rejected": -1.1655998229980469, + "logps/chosen": -2.2206485271453857, + "logps/rejected": -126.70272827148438, + "loss": 0.20282797515392303, + "memory(GiB)": 41.24, + "nll_loss": 0.19769999384880066, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34149640798568726, + "rewards/margins": 11.289339065551758, + "rewards/rejected": -10.947842597961426, + "step": 466, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.6861261261261262, + "grad_norm": 3.721750497817993, + "learning_rate": 5.4400772595816774e-06, + "logits/chosen": -0.7551894783973694, + "logits/rejected": -1.2414366006851196, + "logps/chosen": -1.0196022987365723, + "logps/rejected": -149.0183563232422, + "loss": 0.06770362704992294, + "memory(GiB)": 41.24, + "nll_loss": 0.06111210584640503, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34182730317115784, + "rewards/margins": 13.031250953674316, + "rewards/rejected": -12.689424514770508, + "step": 467, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.691891891891892, + "grad_norm": 0.3983931541442871, + "learning_rate": 5.2346828817197655e-06, + "logits/chosen": -0.844504714012146, + "logits/rejected": -1.2751076221466064, + "logps/chosen": -0.8680186867713928, + "logps/rejected": -140.1030731201172, + "loss": 0.05146872624754906, + "memory(GiB)": 41.24, + "nll_loss": 0.049622949212789536, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3010892868041992, + "rewards/margins": 12.671734809875488, + "rewards/rejected": -12.370645523071289, + "step": 468, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.6976576576576576, + "grad_norm": 0.4564143717288971, + "learning_rate": 5.033136670383554e-06, + "logits/chosen": -0.8619928956031799, + "logits/rejected": -1.2389304637908936, + "logps/chosen": -1.2822086811065674, + "logps/rejected": -125.9210205078125, + "loss": 0.11945895850658417, + "memory(GiB)": 41.24, + "nll_loss": 0.09313815832138062, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6277035474777222, + "rewards/margins": 11.715801239013672, + "rewards/rejected": -11.088098526000977, + "step": 469, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7034234234234233, + "grad_norm": 0.5876050591468811, + "learning_rate": 4.835446809826604e-06, + "logits/chosen": -0.680959165096283, + "logits/rejected": -1.136903166770935, + "logps/chosen": -0.7049185037612915, + "logps/rejected": -144.80047607421875, + "loss": 0.07664791494607925, + "memory(GiB)": 41.24, + "nll_loss": 0.07315680384635925, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3172752559185028, + "rewards/margins": 12.94578742980957, + "rewards/rejected": -12.628510475158691, + "step": 470, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.709189189189189, + "grad_norm": 0.6099125146865845, + "learning_rate": 4.641621327706369e-06, + "logits/chosen": -0.7841821908950806, + "logits/rejected": -1.0943361520767212, + "logps/chosen": -4.46058464050293, + "logps/rejected": -131.05223083496094, + "loss": 0.19772931933403015, + "memory(GiB)": 41.24, + "nll_loss": 0.1958247572183609, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9822390079498291, + "rewards/margins": 12.1292724609375, + "rewards/rejected": -11.14703369140625, + "step": 471, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7149549549549548, + "grad_norm": 0.897890031337738, + "learning_rate": 4.451668094758199e-06, + "logits/chosen": -0.7454246282577515, + "logits/rejected": -1.170987844467163, + "logps/chosen": -1.5386147499084473, + "logps/rejected": -123.76698303222656, + "loss": 0.13460436463356018, + "memory(GiB)": 41.24, + "nll_loss": 0.1309213489294052, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.30889391899108887, + "rewards/margins": 10.850996017456055, + "rewards/rejected": -10.54210090637207, + "step": 472, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7207207207207205, + "grad_norm": 0.7489662170410156, + "learning_rate": 4.265594824475738e-06, + "logits/chosen": -0.6455396413803101, + "logits/rejected": -1.1804579496383667, + "logps/chosen": -1.0532238483428955, + "logps/rejected": -149.65943908691406, + "loss": 0.08711816370487213, + "memory(GiB)": 41.24, + "nll_loss": 0.08352908492088318, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2958245277404785, + "rewards/margins": 13.181537628173828, + "rewards/rejected": -12.885713577270508, + "step": 473, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7264864864864866, + "grad_norm": 0.5648917555809021, + "learning_rate": 4.0834090727977505e-06, + "logits/chosen": -0.8353627920150757, + "logits/rejected": -1.2593697309494019, + "logps/chosen": -1.090264081954956, + "logps/rejected": -136.27279663085938, + "loss": 0.09924523532390594, + "memory(GiB)": 41.24, + "nll_loss": 0.09450168162584305, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22962777316570282, + "rewards/margins": 12.159679412841797, + "rewards/rejected": -11.930051803588867, + "step": 474, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7322522522522523, + "grad_norm": 0.5304217338562012, + "learning_rate": 3.9051182378011755e-06, + "logits/chosen": -0.7211438417434692, + "logits/rejected": -1.2654871940612793, + "logps/chosen": -0.8551865220069885, + "logps/rejected": -130.6512451171875, + "loss": 0.08667407929897308, + "memory(GiB)": 41.24, + "nll_loss": 0.08455201983451843, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3418775796890259, + "rewards/margins": 11.55302619934082, + "rewards/rejected": -11.21114730834961, + "step": 475, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.738018018018018, + "grad_norm": 0.6977109909057617, + "learning_rate": 3.7307295594008472e-06, + "logits/chosen": -0.6623179316520691, + "logits/rejected": -1.1982488632202148, + "logps/chosen": -2.3169212341308594, + "logps/rejected": -138.42564392089844, + "loss": 0.13815613090991974, + "memory(GiB)": 41.24, + "nll_loss": 0.1297193467617035, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3100293278694153, + "rewards/margins": 11.530891418457031, + "rewards/rejected": -11.220861434936523, + "step": 476, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7437837837837837, + "grad_norm": 0.5173090696334839, + "learning_rate": 3.5602501190554193e-06, + "logits/chosen": -0.7255450487136841, + "logits/rejected": -1.2191503047943115, + "logps/chosen": -1.5813761949539185, + "logps/rejected": -128.5985107421875, + "loss": 0.09172145277261734, + "memory(GiB)": 41.24, + "nll_loss": 0.09033821523189545, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34521669149398804, + "rewards/margins": 11.337916374206543, + "rewards/rejected": -10.992700576782227, + "step": 477, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7495495495495494, + "grad_norm": 1.1332759857177734, + "learning_rate": 3.393686839479815e-06, + "logits/chosen": -0.8198791742324829, + "logits/rejected": -1.140008807182312, + "logps/chosen": -2.018730401992798, + "logps/rejected": -119.34492492675781, + "loss": 0.16336697340011597, + "memory(GiB)": 41.24, + "nll_loss": 0.1274120807647705, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.34080076217651367, + "rewards/margins": 10.666831970214844, + "rewards/rejected": -10.326032638549805, + "step": 478, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.755315315315315, + "grad_norm": 0.7781997323036194, + "learning_rate": 3.2310464843641307e-06, + "logits/chosen": -0.7771924734115601, + "logits/rejected": -1.239747166633606, + "logps/chosen": -1.2314516305923462, + "logps/rejected": -144.0001678466797, + "loss": 0.09255998581647873, + "memory(GiB)": 41.24, + "nll_loss": 0.08660240471363068, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3743261992931366, + "rewards/margins": 13.052465438842773, + "rewards/rejected": -12.678139686584473, + "step": 479, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7610810810810813, + "grad_norm": 0.5525214672088623, + "learning_rate": 3.0723356580989904e-06, + "logits/chosen": -0.7069504261016846, + "logits/rejected": -1.2081098556518555, + "logps/chosen": -0.6433517336845398, + "logps/rejected": -138.0076141357422, + "loss": 0.06754517555236816, + "memory(GiB)": 41.24, + "nll_loss": 0.05558161064982414, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16874486207962036, + "rewards/margins": 12.554625511169434, + "rewards/rejected": -12.385881423950195, + "step": 480, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.766846846846847, + "grad_norm": 0.5175738334655762, + "learning_rate": 2.9175608055072913e-06, + "logits/chosen": -0.7529585957527161, + "logits/rejected": -1.2077172994613647, + "logps/chosen": -1.9776062965393066, + "logps/rejected": -140.56031799316406, + "loss": 0.19074459373950958, + "memory(GiB)": 41.24, + "nll_loss": 0.1877429485321045, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3176026940345764, + "rewards/margins": 12.517696380615234, + "rewards/rejected": -12.200093269348145, + "step": 481, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7726126126126127, + "grad_norm": 0.8925073146820068, + "learning_rate": 2.7667282115826033e-06, + "logits/chosen": -0.7878098487854004, + "logits/rejected": -1.2317535877227783, + "logps/chosen": -1.3695169687271118, + "logps/rejected": -134.74041748046875, + "loss": 0.05406276881694794, + "memory(GiB)": 41.24, + "nll_loss": 0.05039753019809723, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3934158682823181, + "rewards/margins": 12.086722373962402, + "rewards/rejected": -11.693306922912598, + "step": 482, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7783783783783784, + "grad_norm": 0.46475109457969666, + "learning_rate": 2.619844001233884e-06, + "logits/chosen": -0.7101900577545166, + "logits/rejected": -1.113659381866455, + "logps/chosen": -2.3404009342193604, + "logps/rejected": -125.89466857910156, + "loss": 0.1386985033750534, + "memory(GiB)": 41.24, + "nll_loss": 0.12376075983047485, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.42396917939186096, + "rewards/margins": 10.65521240234375, + "rewards/rejected": -10.231243133544922, + "step": 483, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.784144144144144, + "grad_norm": 0.7744191288948059, + "learning_rate": 2.4769141390367567e-06, + "logits/chosen": -0.8297570943832397, + "logits/rejected": -1.161577582359314, + "logps/chosen": -2.0823001861572266, + "logps/rejected": -118.72416687011719, + "loss": 0.15259802341461182, + "memory(GiB)": 41.24, + "nll_loss": 0.13629452884197235, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.610589861869812, + "rewards/margins": 10.557499885559082, + "rewards/rejected": -9.946910858154297, + "step": 484, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.78990990990991, + "grad_norm": 0.8081090450286865, + "learning_rate": 2.3379444289913342e-06, + "logits/chosen": -0.8376043438911438, + "logits/rejected": -1.3128516674041748, + "logps/chosen": -1.1590460538864136, + "logps/rejected": -143.44227600097656, + "loss": 0.08370968699455261, + "memory(GiB)": 41.24, + "nll_loss": 0.07294554263353348, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3098699450492859, + "rewards/margins": 12.934307098388672, + "rewards/rejected": -12.62443733215332, + "step": 485, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.7956756756756755, + "grad_norm": 0.4750799536705017, + "learning_rate": 2.2029405142865224e-06, + "logits/chosen": -0.6874585151672363, + "logits/rejected": -1.2795766592025757, + "logps/chosen": -0.8462028503417969, + "logps/rejected": -165.3155517578125, + "loss": 0.094514861702919, + "memory(GiB)": 41.24, + "nll_loss": 0.08393841236829758, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.33502307534217834, + "rewards/margins": 14.392132759094238, + "rewards/rejected": -14.057107925415039, + "step": 486, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.8014414414414412, + "grad_norm": 0.68251633644104, + "learning_rate": 2.0719078770708777e-06, + "logits/chosen": -0.6928752660751343, + "logits/rejected": -1.2014802694320679, + "logps/chosen": -1.1182491779327393, + "logps/rejected": -139.98863220214844, + "loss": 0.08956398069858551, + "memory(GiB)": 41.24, + "nll_loss": 0.08633315563201904, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21974967420101166, + "rewards/margins": 12.46728801727295, + "rewards/rejected": -12.247537612915039, + "step": 487, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.807207207207207, + "grad_norm": 0.5497756600379944, + "learning_rate": 1.9448518382299553e-06, + "logits/chosen": -0.632724940776825, + "logits/rejected": -1.1735813617706299, + "logps/chosen": -1.1711686849594116, + "logps/rejected": -162.91104125976562, + "loss": 0.08067592978477478, + "memory(GiB)": 41.24, + "nll_loss": 0.07910683006048203, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4817790985107422, + "rewards/margins": 14.286357879638672, + "rewards/rejected": -13.804576873779297, + "step": 488, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.812972972972973, + "grad_norm": 0.47427019476890564, + "learning_rate": 1.8217775571702677e-06, + "logits/chosen": -0.7627711892127991, + "logits/rejected": -1.1901848316192627, + "logps/chosen": -1.614353895187378, + "logps/rejected": -155.33387756347656, + "loss": 0.1236029863357544, + "memory(GiB)": 41.24, + "nll_loss": 0.11385863274335861, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27217647433280945, + "rewards/margins": 13.678686141967773, + "rewards/rejected": -13.406510353088379, + "step": 489, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.818738738738739, + "grad_norm": 0.76747065782547, + "learning_rate": 1.7026900316098215e-06, + "logits/chosen": -0.8386929631233215, + "logits/rejected": -1.1703946590423584, + "logps/chosen": -2.046415328979492, + "logps/rejected": -115.5195083618164, + "loss": 0.145877867937088, + "memory(GiB)": 41.24, + "nll_loss": 0.14082565903663635, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.532817542552948, + "rewards/margins": 10.543930053710938, + "rewards/rejected": -10.011112213134766, + "step": 490, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.8245045045045045, + "grad_norm": 1.0751131772994995, + "learning_rate": 1.587594097375078e-06, + "logits/chosen": -0.9069871306419373, + "logits/rejected": -1.2579572200775146, + "logps/chosen": -1.8784408569335938, + "logps/rejected": -120.63430786132812, + "loss": 0.12671029567718506, + "memory(GiB)": 41.24, + "nll_loss": 0.11608563363552094, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.49139508605003357, + "rewards/margins": 10.863404273986816, + "rewards/rejected": -10.372008323669434, + "step": 491, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.8302702702702702, + "grad_norm": 0.6973512768745422, + "learning_rate": 1.4764944282046445e-06, + "logits/chosen": -0.7752227187156677, + "logits/rejected": -1.1322190761566162, + "logps/chosen": -2.4691591262817383, + "logps/rejected": -119.4776382446289, + "loss": 0.1543593853712082, + "memory(GiB)": 41.24, + "nll_loss": 0.1450590342283249, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4503910541534424, + "rewards/margins": 10.80586051940918, + "rewards/rejected": -10.355469703674316, + "step": 492, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.836036036036036, + "grad_norm": 1.0350126028060913, + "learning_rate": 1.36939553555947e-06, + "logits/chosen": -0.7532844543457031, + "logits/rejected": -1.1732583045959473, + "logps/chosen": -1.253281593322754, + "logps/rejected": -133.32554626464844, + "loss": 0.07413357496261597, + "memory(GiB)": 41.24, + "nll_loss": 0.06545071303844452, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2877812385559082, + "rewards/margins": 11.76184368133545, + "rewards/rejected": -11.4740629196167, + "step": 493, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.8418018018018016, + "grad_norm": 0.6703996658325195, + "learning_rate": 1.2663017684396593e-06, + "logits/chosen": -0.6817619800567627, + "logits/rejected": -1.2400188446044922, + "logps/chosen": -0.7767384052276611, + "logps/rejected": -167.19970703125, + "loss": 0.06508862227201462, + "memory(GiB)": 41.24, + "nll_loss": 0.06348208338022232, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2484712302684784, + "rewards/margins": 14.862464904785156, + "rewards/rejected": -14.613993644714355, + "step": 494, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.847567567567568, + "grad_norm": 0.47343385219573975, + "learning_rate": 1.167217313207858e-06, + "logits/chosen": -0.7568473815917969, + "logits/rejected": -1.195677399635315, + "logps/chosen": -1.6267422437667847, + "logps/rejected": -131.75537109375, + "loss": 0.12443436682224274, + "memory(GiB)": 41.24, + "nll_loss": 0.09817352145910263, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5396956205368042, + "rewards/margins": 11.83067512512207, + "rewards/rejected": -11.290980339050293, + "step": 495, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.8533333333333335, + "grad_norm": 0.9057773947715759, + "learning_rate": 1.0721461934192545e-06, + "logits/chosen": -0.6768568754196167, + "logits/rejected": -1.2324684858322144, + "logps/chosen": -1.2529605627059937, + "logps/rejected": -163.7919464111328, + "loss": 0.10538756847381592, + "memory(GiB)": 41.24, + "nll_loss": 0.10154788941144943, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2912774980068207, + "rewards/margins": 14.406535148620605, + "rewards/rejected": -14.115256309509277, + "step": 496, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.859099099099099, + "grad_norm": 0.7119655609130859, + "learning_rate": 9.810922696582014e-07, + "logits/chosen": -0.7816168069839478, + "logits/rejected": -1.242786169052124, + "logps/chosen": -2.132449150085449, + "logps/rejected": -135.7627716064453, + "loss": 0.21441125869750977, + "memory(GiB)": 41.24, + "nll_loss": 0.21072225272655487, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8211126327514648, + "rewards/margins": 12.505435943603516, + "rewards/rejected": -11.684324264526367, + "step": 497, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.864864864864865, + "grad_norm": 1.0358115434646606, + "learning_rate": 8.940592393814728e-07, + "logits/chosen": -0.7968149781227112, + "logits/rejected": -1.126916527748108, + "logps/chosen": -1.569638967514038, + "logps/rejected": -114.44811248779297, + "loss": 0.11331821978092194, + "memory(GiB)": 41.24, + "nll_loss": 0.09483326226472855, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2961835563182831, + "rewards/margins": 10.28332233428955, + "rewards/rejected": -9.987138748168945, + "step": 498, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.8706306306306306, + "grad_norm": 0.49830037355422974, + "learning_rate": 8.110506367680515e-07, + "logits/chosen": -0.7517850995063782, + "logits/rejected": -1.1779118776321411, + "logps/chosen": -1.2592693567276, + "logps/rejected": -130.19686889648438, + "loss": 0.12627199292182922, + "memory(GiB)": 41.24, + "nll_loss": 0.11919055879116058, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3111531734466553, + "rewards/margins": 11.515169143676758, + "rewards/rejected": -11.204015731811523, + "step": 499, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.8763963963963963, + "grad_norm": 0.7826987504959106, + "learning_rate": 7.320698325756658e-07, + "logits/chosen": -0.780491054058075, + "logits/rejected": -1.2717281579971313, + "logps/chosen": -0.5362758040428162, + "logps/rejected": -147.2077178955078, + "loss": 0.05917176604270935, + "memory(GiB)": 41.24, + "nll_loss": 0.0573415532708168, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16316349804401398, + "rewards/margins": 12.925711631774902, + "rewards/rejected": -12.76254653930664, + "step": 500, + "train_speed(iter/s)": 0.010721 + }, + { + "epoch": 2.882162162162162, + "grad_norm": 0.3995133340358734, + "learning_rate": 6.571200340039218e-07, + "logits/chosen": -0.7449495792388916, + "logits/rejected": -1.2002133131027222, + "logps/chosen": -0.8479886054992676, + "logps/rejected": -145.81724548339844, + "loss": 0.07765054702758789, + "memory(GiB)": 41.24, + "nll_loss": 0.07375539839267731, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2873612940311432, + "rewards/margins": 12.844383239746094, + "rewards/rejected": -12.557022094726562, + "step": 501, + "train_speed(iter/s)": 0.010717 + }, + { + "epoch": 2.8879279279279277, + "grad_norm": 0.5732494592666626, + "learning_rate": 5.862042845640403e-07, + "logits/chosen": -0.7512827515602112, + "logits/rejected": -1.11747407913208, + "logps/chosen": -1.462705135345459, + "logps/rejected": -119.50070190429688, + "loss": 0.10571929067373276, + "memory(GiB)": 41.24, + "nll_loss": 0.09841780364513397, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.36830925941467285, + "rewards/margins": 10.630249977111816, + "rewards/rejected": -10.261940956115723, + "step": 502, + "train_speed(iter/s)": 0.010717 + }, + { + "epoch": 2.8936936936936934, + "grad_norm": 0.64256751537323, + "learning_rate": 5.19325463955278e-07, + "logits/chosen": -0.8329352140426636, + "logits/rejected": -1.2823573350906372, + "logps/chosen": -1.3889741897583008, + "logps/rejected": -134.369384765625, + "loss": 0.08437025547027588, + "memory(GiB)": 41.24, + "nll_loss": 0.08264239132404327, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4939664602279663, + "rewards/margins": 11.983332633972168, + "rewards/rejected": -11.48936653137207, + "step": 503, + "train_speed(iter/s)": 0.010717 + }, + { + "epoch": 2.8994594594594596, + "grad_norm": 0.7437162399291992, + "learning_rate": 4.564862879479881e-07, + "logits/chosen": -0.8067994117736816, + "logits/rejected": -1.1400036811828613, + "logps/chosen": -1.5383354425430298, + "logps/rejected": -98.28523254394531, + "loss": 0.1585976630449295, + "memory(GiB)": 41.24, + "nll_loss": 0.15575863420963287, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5421609878540039, + "rewards/margins": 9.232922554016113, + "rewards/rejected": -8.69076156616211, + "step": 504, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.9052252252252253, + "grad_norm": 0.7169486880302429, + "learning_rate": 3.976893082733413e-07, + "logits/chosen": -0.7986385226249695, + "logits/rejected": -1.2822511196136475, + "logps/chosen": -0.523093581199646, + "logps/rejected": -128.82168579101562, + "loss": 0.062391187995672226, + "memory(GiB)": 41.24, + "nll_loss": 0.05369999632239342, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26059597730636597, + "rewards/margins": 11.467589378356934, + "rewards/rejected": -11.206992149353027, + "step": 505, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.910990990990991, + "grad_norm": 0.5572133660316467, + "learning_rate": 3.429369125197091e-07, + "logits/chosen": -0.8438900113105774, + "logits/rejected": -1.257434606552124, + "logps/chosen": -1.2440414428710938, + "logps/rejected": -126.9652328491211, + "loss": 0.12426011264324188, + "memory(GiB)": 41.24, + "nll_loss": 0.10738712549209595, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3913893401622772, + "rewards/margins": 11.357544898986816, + "rewards/rejected": -10.966154098510742, + "step": 506, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.9167567567567567, + "grad_norm": 0.8767194747924805, + "learning_rate": 2.9223132403570773e-07, + "logits/chosen": -0.756206750869751, + "logits/rejected": -1.2638530731201172, + "logps/chosen": -1.812301754951477, + "logps/rejected": -142.83883666992188, + "loss": 0.11816136538982391, + "memory(GiB)": 41.24, + "nll_loss": 0.11462105065584183, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.411607027053833, + "rewards/margins": 12.546673774719238, + "rewards/rejected": -12.1350679397583, + "step": 507, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.9225225225225224, + "grad_norm": 0.7405654788017273, + "learning_rate": 2.45574601839893e-07, + "logits/chosen": -0.7418671250343323, + "logits/rejected": -1.1688292026519775, + "logps/chosen": -0.9982606172561646, + "logps/rejected": -118.87340545654297, + "loss": 0.0927569791674614, + "memory(GiB)": 41.24, + "nll_loss": 0.07881642878055573, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.39366358518600464, + "rewards/margins": 10.498245239257812, + "rewards/rejected": -10.104582786560059, + "step": 508, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.928288288288288, + "grad_norm": 0.8121258020401001, + "learning_rate": 2.0296864053721555e-07, + "logits/chosen": -0.6867554783821106, + "logits/rejected": -1.1909139156341553, + "logps/chosen": -0.5226776003837585, + "logps/rejected": -135.34002685546875, + "loss": 0.07622242718935013, + "memory(GiB)": 41.24, + "nll_loss": 0.03218510001897812, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22311460971832275, + "rewards/margins": 11.750666618347168, + "rewards/rejected": -11.527551651000977, + "step": 509, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.9340540540540543, + "grad_norm": 1.2022919654846191, + "learning_rate": 1.6441517024200492e-07, + "logits/chosen": -0.9085918068885803, + "logits/rejected": -1.2135717868804932, + "logps/chosen": -2.024538993835449, + "logps/rejected": -106.9240493774414, + "loss": 0.19892165064811707, + "memory(GiB)": 41.24, + "nll_loss": 0.18685536086559296, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.448761910200119, + "rewards/margins": 9.915888786315918, + "rewards/rejected": -9.467126846313477, + "step": 510, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.93981981981982, + "grad_norm": 1.1970798969268799, + "learning_rate": 1.299157565077702e-07, + "logits/chosen": -0.7311093807220459, + "logits/rejected": -1.2175939083099365, + "logps/chosen": -1.162784457206726, + "logps/rejected": -158.7892303466797, + "loss": 0.11232640594244003, + "memory(GiB)": 41.24, + "nll_loss": 0.10971198230981827, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20378267765045166, + "rewards/margins": 13.85474967956543, + "rewards/rejected": -13.65096664428711, + "step": 511, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.9455855855855857, + "grad_norm": 0.5884090065956116, + "learning_rate": 9.947180026357305e-08, + "logits/chosen": -0.8717092275619507, + "logits/rejected": -1.2725342512130737, + "logps/chosen": -2.4550178050994873, + "logps/rejected": -148.26522827148438, + "loss": 0.10548409074544907, + "memory(GiB)": 41.24, + "nll_loss": 0.09250208735466003, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.456002801656723, + "rewards/margins": 13.166388511657715, + "rewards/rejected": -12.7103853225708, + "step": 512, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.9513513513513514, + "grad_norm": 1.0819729566574097, + "learning_rate": 7.308453775721758e-08, + "logits/chosen": -0.8463583588600159, + "logits/rejected": -1.2036621570587158, + "logps/chosen": -2.860271692276001, + "logps/rejected": -130.79127502441406, + "loss": 0.2330038994550705, + "memory(GiB)": 41.24, + "nll_loss": 0.19909462332725525, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.3324648141860962, + "rewards/margins": 11.67158317565918, + "rewards/rejected": -11.339119911193848, + "step": 513, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.957117117117117, + "grad_norm": 2.916995048522949, + "learning_rate": 5.075504050499058e-08, + "logits/chosen": -0.8122426867485046, + "logits/rejected": -1.2027369737625122, + "logps/chosen": -2.2083566188812256, + "logps/rejected": -124.13884735107422, + "loss": 0.11903891712427139, + "memory(GiB)": 41.24, + "nll_loss": 0.11742937564849854, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.39628690481185913, + "rewards/margins": 11.131566047668457, + "rewards/rejected": -10.735278129577637, + "step": 514, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.962882882882883, + "grad_norm": 0.6064572930335999, + "learning_rate": 3.2484215248140824e-08, + "logits/chosen": -0.8155410885810852, + "logits/rejected": -1.1723310947418213, + "logps/chosen": -2.320070266723633, + "logps/rejected": -124.64006042480469, + "loss": 0.2072904109954834, + "memory(GiB)": 41.24, + "nll_loss": 0.17831988632678986, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3959542214870453, + "rewards/margins": 11.01950454711914, + "rewards/rejected": -10.623550415039062, + "step": 515, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.9686486486486485, + "grad_norm": 1.028844952583313, + "learning_rate": 1.827280391611952e-08, + "logits/chosen": -0.795734167098999, + "logits/rejected": -1.1955606937408447, + "logps/chosen": -2.1585350036621094, + "logps/rejected": -132.4622039794922, + "loss": 0.17623259127140045, + "memory(GiB)": 41.24, + "nll_loss": 0.13708879053592682, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.5354299545288086, + "rewards/margins": 11.798476219177246, + "rewards/rejected": -11.263046264648438, + "step": 516, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.9744144144144142, + "grad_norm": 0.8039427995681763, + "learning_rate": 8.121383596393362e-09, + "logits/chosen": -0.7880966663360596, + "logits/rejected": -1.1564136743545532, + "logps/chosen": -3.502894163131714, + "logps/rejected": -132.29896545410156, + "loss": 0.21312487125396729, + "memory(GiB)": 41.24, + "nll_loss": 0.1841593086719513, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.5323241949081421, + "rewards/margins": 12.035751342773438, + "rewards/rejected": -11.503427505493164, + "step": 517, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.98018018018018, + "grad_norm": 0.8289259672164917, + "learning_rate": 2.0303665110410484e-09, + "logits/chosen": -0.8445495367050171, + "logits/rejected": -1.2218854427337646, + "logps/chosen": -1.0164295434951782, + "logps/rejected": -128.60525512695312, + "loss": 0.10160036385059357, + "memory(GiB)": 41.24, + "nll_loss": 0.07469040155410767, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.6711289882659912, + "rewards/margins": 11.901240348815918, + "rewards/rejected": -11.230112075805664, + "step": 518, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.985945945945946, + "grad_norm": 0.5789499878883362, + "learning_rate": 0.0, + "logits/chosen": -0.7110620737075806, + "logits/rejected": -1.2225096225738525, + "logps/chosen": -1.0744678974151611, + "logps/rejected": -156.6178436279297, + "loss": 0.05970888212323189, + "memory(GiB)": 41.24, + "nll_loss": 0.05469948798418045, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14128968119621277, + "rewards/margins": 13.778985023498535, + "rewards/rejected": -13.637697219848633, + "step": 519, + "train_speed(iter/s)": 0.010718 + }, + { + "epoch": 2.985945945945946, + "eval_logits/chosen": -0.7823337912559509, + "eval_logits/rejected": -1.1537983417510986, + "eval_logps/chosen": -5.033455848693848, + "eval_logps/rejected": -104.95735931396484, + "eval_loss": 0.5656055808067322, + "eval_nll_loss": 0.4618266224861145, + "eval_rewards/accuracies": 0.9642857313156128, + "eval_rewards/chosen": 0.18145908415317535, + "eval_rewards/margins": 8.959060668945312, + "eval_rewards/rejected": -8.777602195739746, + "eval_runtime": 64.1967, + "eval_samples_per_second": 0.872, + "eval_steps_per_second": 0.436, + "step": 519 + } + ], + "logging_steps": 1, + "max_steps": 519, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.8346139128365056e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}