File size: 61,091 Bytes
b5ae70e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 | {
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.18450184501845018,
"eval_steps": 500,
"global_step": 50,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"cispo_clip_ratio": 0.364774439483881,
"completions/clipped_ratio": 1.0,
"completions/max_length": 5409.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 2607.8125,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 674.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.4863766431808472,
"epoch": 0.0036900369003690036,
"frac_reward_zero_std": 0.0,
"grad_norm": 1.6501518985023722e-05,
"learning_rate": 0.0002,
"loss": -5.250487447483465e-05,
"num_tokens": 45661.0,
"reward": 0.2993389666080475,
"reward_std": 0.17980147898197174,
"rewards/humor_reward/mean": 0.2993389666080475,
"rewards/humor_reward/std": 0.17980147898197174,
"sampling/importance_sampling_ratio/max": 0.0032514820341020823,
"sampling/importance_sampling_ratio/mean": 0.00025803607422858477,
"sampling/importance_sampling_ratio/min": 6.591494638996664e-08,
"sampling/sampling_logp_difference/max": 3.1304593086242676,
"sampling/sampling_logp_difference/mean": 0.04722540080547333,
"step": 1,
"step_time": 79.16197230700345
},
{
"cispo_clip_ratio": 0.4739677682518959,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 4785.0,
"completions/max_terminated_length": 439.0,
"completions/mean_length": 2616.0625,
"completions/mean_terminated_length": 439.0,
"completions/min_length": 439.0,
"completions/min_terminated_length": 439.0,
"entropy": 1.2316511571407318,
"epoch": 0.007380073800738007,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.003729997668415308,
"learning_rate": 0.0002,
"loss": -0.003799113677814603,
"num_tokens": 91230.0,
"reward": 0.5310899019241333,
"reward_std": 0.27845898270606995,
"rewards/humor_reward/mean": 0.5310899019241333,
"rewards/humor_reward/std": 0.27845898270606995,
"sampling/importance_sampling_ratio/max": 1.048789381980896,
"sampling/importance_sampling_ratio/mean": 0.12509621679782867,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.2385149002075195,
"sampling/sampling_logp_difference/mean": 0.04188045486807823,
"step": 2,
"step_time": 66.1825476239901
},
{
"cispo_clip_ratio": 0.5831195935606956,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 2244.0,
"completions/mean_length": 3197.3125,
"completions/mean_terminated_length": 2244.0,
"completions/min_length": 1313.0,
"completions/min_terminated_length": 2244.0,
"entropy": 1.3326809257268906,
"epoch": 0.01107011070110701,
"frac_reward_zero_std": 0.0,
"grad_norm": 5.462636181619018e-05,
"learning_rate": 0.0002,
"loss": 0.00016624285490252078,
"num_tokens": 146995.0,
"reward": 0.4853614568710327,
"reward_std": 0.2525769770145416,
"rewards/humor_reward/mean": 0.4853614568710327,
"rewards/humor_reward/std": 0.252577006816864,
"sampling/importance_sampling_ratio/max": 0.022306665778160095,
"sampling/importance_sampling_ratio/mean": 0.001404650043696165,
"sampling/importance_sampling_ratio/min": 1.7064398505350908e-10,
"sampling/sampling_logp_difference/max": 3.054729700088501,
"sampling/sampling_logp_difference/mean": 0.0515315905213356,
"step": 3,
"step_time": 86.2583410259831
},
{
"cispo_clip_ratio": 0.5900484155863523,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3705.3125,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1009.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.533411294221878,
"epoch": 0.014760147601476014,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0004810348036698997,
"learning_rate": 0.0002,
"loss": -0.0014380415668711066,
"num_tokens": 210440.0,
"reward": 0.48688405752182007,
"reward_std": 0.2839711010456085,
"rewards/humor_reward/mean": 0.48688405752182007,
"rewards/humor_reward/std": 0.2839711010456085,
"sampling/importance_sampling_ratio/max": 0.21202650666236877,
"sampling/importance_sampling_ratio/mean": 0.013267980888485909,
"sampling/importance_sampling_ratio/min": 1.5972070888103929e-10,
"sampling/sampling_logp_difference/max": 3.0988059043884277,
"sampling/sampling_logp_difference/mean": 0.04585312306880951,
"step": 4,
"step_time": 83.59175038301328
},
{
"cispo_clip_ratio": 0.6057924032211304,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4057.8125,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 482.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.3052150011062622,
"epoch": 0.01845018450184502,
"frac_reward_zero_std": 0.0,
"grad_norm": 4.325129793869564e-06,
"learning_rate": 0.0002,
"loss": -5.259538738755509e-05,
"num_tokens": 279525.0,
"reward": 0.409991055727005,
"reward_std": 0.27515894174575806,
"rewards/humor_reward/mean": 0.409991055727005,
"rewards/humor_reward/std": 0.27515894174575806,
"sampling/importance_sampling_ratio/max": 0.0020468428265303373,
"sampling/importance_sampling_ratio/mean": 0.0002199342561652884,
"sampling/importance_sampling_ratio/min": 4.834557643107473e-09,
"sampling/sampling_logp_difference/max": 1.898941993713379,
"sampling/sampling_logp_difference/mean": 0.04034310206770897,
"step": 5,
"step_time": 63.17914728100004
},
{
"cispo_clip_ratio": 0.51509914919734,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4054.625,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 106.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.3227798640727997,
"epoch": 0.02214022140221402,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0003930912062060088,
"learning_rate": 0.0002,
"loss": -8.775031892582774e-05,
"num_tokens": 348111.0,
"reward": 0.47100865840911865,
"reward_std": 0.2912721335887909,
"rewards/humor_reward/mean": 0.47100865840911865,
"rewards/humor_reward/std": 0.2912721633911133,
"sampling/importance_sampling_ratio/max": 0.26060751080513,
"sampling/importance_sampling_ratio/mean": 0.0164032019674778,
"sampling/importance_sampling_ratio/min": 9.639339326739105e-10,
"sampling/sampling_logp_difference/max": 2.0555481910705566,
"sampling/sampling_logp_difference/mean": 0.039414241909980774,
"step": 6,
"step_time": 82.86594534001779
},
{
"cispo_clip_ratio": 0.6096626706421375,
"completions/clipped_ratio": 0.875,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 3121.0,
"completions/mean_length": 3594.3125,
"completions/mean_terminated_length": 1685.5,
"completions/min_length": 184.0,
"completions/min_terminated_length": 250.0,
"entropy": 1.4805490970611572,
"epoch": 0.025830258302583026,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0001002174976747483,
"learning_rate": 0.0002,
"loss": -0.00036368955625221133,
"num_tokens": 409556.0,
"reward": 0.25648510456085205,
"reward_std": 0.21258896589279175,
"rewards/humor_reward/mean": 0.25648510456085205,
"rewards/humor_reward/std": 0.21258896589279175,
"sampling/importance_sampling_ratio/max": 0.042766377329826355,
"sampling/importance_sampling_ratio/mean": 0.003916537389159203,
"sampling/importance_sampling_ratio/min": 3.151466310136186e-10,
"sampling/sampling_logp_difference/max": 2.511254072189331,
"sampling/sampling_logp_difference/mean": 0.04306597262620926,
"step": 7,
"step_time": 77.88916695199441
},
{
"cispo_clip_ratio": 0.4443662669509649,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4063.8125,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 426.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.275453269481659,
"epoch": 0.02952029520295203,
"frac_reward_zero_std": 0.0,
"grad_norm": 1.6524854800081812e-05,
"learning_rate": 0.0002,
"loss": 6.745757855242118e-05,
"num_tokens": 478513.0,
"reward": 0.3238402009010315,
"reward_std": 0.2930999994277954,
"rewards/humor_reward/mean": 0.3238402009010315,
"rewards/humor_reward/std": 0.2931000292301178,
"sampling/importance_sampling_ratio/max": 0.010375253856182098,
"sampling/importance_sampling_ratio/mean": 0.0010308868950232863,
"sampling/importance_sampling_ratio/min": 5.8848907968922504e-08,
"sampling/sampling_logp_difference/max": 3.019871711730957,
"sampling/sampling_logp_difference/mean": 0.038398630917072296,
"step": 8,
"step_time": 62.586684064008296
},
{
"cispo_clip_ratio": 0.4614889621734619,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3351.5,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 925.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.392012134194374,
"epoch": 0.033210332103321034,
"frac_reward_zero_std": 0.0,
"grad_norm": 5.4697693485650234e-08,
"learning_rate": 0.0002,
"loss": 3.4436376949997793e-07,
"num_tokens": 536969.0,
"reward": 0.320010781288147,
"reward_std": 0.26967623829841614,
"rewards/humor_reward/mean": 0.320010781288147,
"rewards/humor_reward/std": 0.26967623829841614,
"sampling/importance_sampling_ratio/max": 2.8325872335699387e-05,
"sampling/importance_sampling_ratio/mean": 4.4159696699352935e-06,
"sampling/importance_sampling_ratio/min": 4.985558949184565e-13,
"sampling/sampling_logp_difference/max": 2.7537732124328613,
"sampling/sampling_logp_difference/mean": 0.05181171000003815,
"step": 9,
"step_time": 85.44417907499883
},
{
"cispo_clip_ratio": 0.46727752313017845,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4545.25,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1297.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.277551643550396,
"epoch": 0.03690036900369004,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0018553344998508692,
"learning_rate": 0.0002,
"loss": -0.00018077026470564306,
"num_tokens": 613629.0,
"reward": 0.3976645767688751,
"reward_std": 0.3508976995944977,
"rewards/humor_reward/mean": 0.3976645767688751,
"rewards/humor_reward/std": 0.3508976995944977,
"sampling/importance_sampling_ratio/max": 0.32763394713401794,
"sampling/importance_sampling_ratio/mean": 0.02203518897294998,
"sampling/importance_sampling_ratio/min": 2.647046748460724e-10,
"sampling/sampling_logp_difference/max": 3.727196216583252,
"sampling/sampling_logp_difference/mean": 0.037366170436143875,
"step": 10,
"step_time": 83.16811528199469
},
{
"cispo_clip_ratio": 0.564013222232461,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4322.8125,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 705.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.0119460746645927,
"epoch": 0.04059040590405904,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0012440788559615612,
"learning_rate": 0.0002,
"loss": -0.0035373736172914505,
"num_tokens": 687402.0,
"reward": 0.378947377204895,
"reward_std": 0.2340850830078125,
"rewards/humor_reward/mean": 0.378947377204895,
"rewards/humor_reward/std": 0.2340850979089737,
"sampling/importance_sampling_ratio/max": 0.30296310782432556,
"sampling/importance_sampling_ratio/mean": 0.018969321623444557,
"sampling/importance_sampling_ratio/min": 8.10443156811988e-15,
"sampling/sampling_logp_difference/max": 3.91379976272583,
"sampling/sampling_logp_difference/mean": 0.04132331907749176,
"step": 11,
"step_time": 94.6180467310187
},
{
"cispo_clip_ratio": 0.3971639759838581,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 5086.0,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1271.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.0960774347186089,
"epoch": 0.04428044280442804,
"frac_reward_zero_std": 0.0,
"grad_norm": 5.153976962901652e-05,
"learning_rate": 0.0002,
"loss": -0.00044693646486848593,
"num_tokens": 772714.0,
"reward": 0.32133913040161133,
"reward_std": 0.380583792924881,
"rewards/humor_reward/mean": 0.32133913040161133,
"rewards/humor_reward/std": 0.38058382272720337,
"sampling/importance_sampling_ratio/max": 0.017419712617993355,
"sampling/importance_sampling_ratio/mean": 0.001832809066399932,
"sampling/importance_sampling_ratio/min": 6.273771866599498e-12,
"sampling/sampling_logp_difference/max": 5.7523579597473145,
"sampling/sampling_logp_difference/mean": 0.034892238676548004,
"step": 12,
"step_time": 79.48709479898389
},
{
"cispo_clip_ratio": 0.5508307814598083,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4785.875,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1482.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.299708716571331,
"epoch": 0.04797047970479705,
"frac_reward_zero_std": 0.0,
"grad_norm": 6.673526968370425e-06,
"learning_rate": 0.0002,
"loss": -2.0674373445217498e-05,
"num_tokens": 853224.0,
"reward": 0.36821186542510986,
"reward_std": 0.31694290041923523,
"rewards/humor_reward/mean": 0.36821186542510986,
"rewards/humor_reward/std": 0.3169429302215576,
"sampling/importance_sampling_ratio/max": 0.0009282280807383358,
"sampling/importance_sampling_ratio/mean": 0.0001542143290862441,
"sampling/importance_sampling_ratio/min": 4.797664487909969e-10,
"sampling/sampling_logp_difference/max": 2.7313647270202637,
"sampling/sampling_logp_difference/mean": 0.03830663487315178,
"step": 13,
"step_time": 82.41276069695596
},
{
"cispo_clip_ratio": 0.3803188279271126,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 1625.0,
"completions/mean_length": 4955.6875,
"completions/mean_terminated_length": 1625.0,
"completions/min_length": 968.0,
"completions/min_terminated_length": 1625.0,
"entropy": 1.364896483719349,
"epoch": 0.05166051660516605,
"frac_reward_zero_std": 0.0,
"grad_norm": 1.3385028978518676e-05,
"learning_rate": 0.0002,
"loss": 2.8572056180564687e-05,
"num_tokens": 937123.0,
"reward": 0.2386319488286972,
"reward_std": 0.28499212861061096,
"rewards/humor_reward/mean": 0.2386319488286972,
"rewards/humor_reward/std": 0.28499215841293335,
"sampling/importance_sampling_ratio/max": 0.009222447872161865,
"sampling/importance_sampling_ratio/mean": 0.0005764853558503091,
"sampling/importance_sampling_ratio/min": 1.6275064690298314e-15,
"sampling/sampling_logp_difference/max": 4.84999942779541,
"sampling/sampling_logp_difference/mean": 0.04662536829710007,
"step": 14,
"step_time": 85.49535570498847
},
{
"cispo_clip_ratio": 0.3275146186351776,
"completions/clipped_ratio": 0.875,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 2590.0,
"completions/mean_length": 4530.75,
"completions/mean_terminated_length": 2166.5,
"completions/min_length": 505.0,
"completions/min_terminated_length": 1743.0,
"entropy": 0.9317370727658272,
"epoch": 0.055350553505535055,
"frac_reward_zero_std": 0.0,
"grad_norm": 4.116483614780009e-05,
"learning_rate": 0.0002,
"loss": 0.0002636128047015518,
"num_tokens": 1013327.0,
"reward": 0.2526451349258423,
"reward_std": 0.36080095171928406,
"rewards/humor_reward/mean": 0.2526451349258423,
"rewards/humor_reward/std": 0.36080095171928406,
"sampling/importance_sampling_ratio/max": 0.011370004154741764,
"sampling/importance_sampling_ratio/mean": 0.001944657531566918,
"sampling/importance_sampling_ratio/min": 1.1834137185820492e-12,
"sampling/sampling_logp_difference/max": 3.4003725051879883,
"sampling/sampling_logp_difference/mean": 0.03138995170593262,
"step": 15,
"step_time": 73.45879589200194
},
{
"cispo_clip_ratio": 0.1871738387271762,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 5333.125,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 854.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.1076993495225906,
"epoch": 0.05904059040590406,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0002949201443698257,
"learning_rate": 0.0002,
"loss": 0.0002496525994502008,
"num_tokens": 1102593.0,
"reward": 0.12923595309257507,
"reward_std": 0.24049529433250427,
"rewards/humor_reward/mean": 0.12923595309257507,
"rewards/humor_reward/std": 0.24049529433250427,
"sampling/importance_sampling_ratio/max": 0.13862203061580658,
"sampling/importance_sampling_ratio/mean": 0.013717259280383587,
"sampling/importance_sampling_ratio/min": 2.0222361563071445e-09,
"sampling/sampling_logp_difference/max": 1.9613001346588135,
"sampling/sampling_logp_difference/mean": 0.02957136556506157,
"step": 16,
"step_time": 76.10329114498745
},
{
"cispo_clip_ratio": 0.2707713171839714,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 5502.0,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 2296.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.3415134847164154,
"epoch": 0.06273062730627306,
"frac_reward_zero_std": 0.0,
"grad_norm": 1.443507812837197e-06,
"learning_rate": 0.0002,
"loss": 1.1968148101004772e-05,
"num_tokens": 1194337.0,
"reward": 0.14995136857032776,
"reward_std": 0.2515668570995331,
"rewards/humor_reward/mean": 0.14995136857032776,
"rewards/humor_reward/std": 0.25156688690185547,
"sampling/importance_sampling_ratio/max": 0.00044680986320599914,
"sampling/importance_sampling_ratio/mean": 3.927269790438004e-05,
"sampling/importance_sampling_ratio/min": 9.991393268293791e-10,
"sampling/sampling_logp_difference/max": 3.0433225631713867,
"sampling/sampling_logp_difference/mean": 0.033172741532325745,
"step": 17,
"step_time": 76.20640446299512
},
{
"cispo_clip_ratio": 0.4475329667329788,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4663.4375,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1583.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.0032905638217926,
"epoch": 0.06642066420664207,
"frac_reward_zero_std": 0.0,
"grad_norm": 4.456151003751074e-08,
"learning_rate": 0.0002,
"loss": 1.6434560734523984e-07,
"num_tokens": 1273560.0,
"reward": 0.47607260942459106,
"reward_std": 0.3206636905670166,
"rewards/humor_reward/mean": 0.47607260942459106,
"rewards/humor_reward/std": 0.3206636905670166,
"sampling/importance_sampling_ratio/max": 1.8386488591204397e-05,
"sampling/importance_sampling_ratio/mean": 2.1001696950406767e-06,
"sampling/importance_sampling_ratio/min": 2.987033566998441e-14,
"sampling/sampling_logp_difference/max": 2.9727420806884766,
"sampling/sampling_logp_difference/mean": 0.04333854466676712,
"step": 18,
"step_time": 91.44482385799347
},
{
"cispo_clip_ratio": 0.5934446323662996,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4332.0,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 692.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.2455691695213318,
"epoch": 0.07011070110701106,
"frac_reward_zero_std": 0.0,
"grad_norm": 2.6948493541567586e-05,
"learning_rate": 0.0002,
"loss": 0.00015589207760058343,
"num_tokens": 1346808.0,
"reward": 0.4407455325126648,
"reward_std": 0.32844167947769165,
"rewards/humor_reward/mean": 0.4407455325126648,
"rewards/humor_reward/std": 0.32844167947769165,
"sampling/importance_sampling_ratio/max": 0.0622418075799942,
"sampling/importance_sampling_ratio/mean": 0.00455310195684433,
"sampling/importance_sampling_ratio/min": 6.87965170942384e-11,
"sampling/sampling_logp_difference/max": 1.9999836683273315,
"sampling/sampling_logp_difference/mean": 0.040162429213523865,
"step": 19,
"step_time": 63.66296142600186
},
{
"cispo_clip_ratio": 0.3904235363006592,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 3539.0,
"completions/mean_length": 4865.125,
"completions/mean_terminated_length": 3539.0,
"completions/min_length": 1705.0,
"completions/min_terminated_length": 3539.0,
"entropy": 1.1548645570874214,
"epoch": 0.07380073800738007,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0003063328331336379,
"learning_rate": 0.0002,
"loss": 0.0016583555843681097,
"num_tokens": 1428810.0,
"reward": 0.32657718658447266,
"reward_std": 0.3702032268047333,
"rewards/humor_reward/mean": 0.32657718658447266,
"rewards/humor_reward/std": 0.37020325660705566,
"sampling/importance_sampling_ratio/max": 0.07926318794488907,
"sampling/importance_sampling_ratio/mean": 0.00835469737648964,
"sampling/importance_sampling_ratio/min": 1.397608049256982e-12,
"sampling/sampling_logp_difference/max": 3.58780574798584,
"sampling/sampling_logp_difference/mean": 0.036592792719602585,
"step": 20,
"step_time": 80.32529347503441
},
{
"cispo_clip_ratio": 0.46490118466317654,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4817.375,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1635.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.468906119465828,
"epoch": 0.07749077490774908,
"frac_reward_zero_std": 0.0,
"grad_norm": 9.814787915729539e-08,
"learning_rate": 0.0002,
"loss": -6.973982635827269e-07,
"num_tokens": 1510048.0,
"reward": 0.4167077839374542,
"reward_std": 0.3755662441253662,
"rewards/humor_reward/mean": 0.4167077839374542,
"rewards/humor_reward/std": 0.3755662441253662,
"sampling/importance_sampling_ratio/max": 2.670207504706923e-05,
"sampling/importance_sampling_ratio/mean": 3.46238084603101e-06,
"sampling/importance_sampling_ratio/min": 6.556125781154165e-11,
"sampling/sampling_logp_difference/max": 2.0220751762390137,
"sampling/sampling_logp_difference/mean": 0.04112354293465614,
"step": 21,
"step_time": 66.39175808998698
},
{
"cispo_clip_ratio": 0.4479520544409752,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4476.625,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 576.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.3538827523589134,
"epoch": 0.08118081180811808,
"frac_reward_zero_std": 0.0,
"grad_norm": 9.053791472979356e-06,
"learning_rate": 0.0002,
"loss": -4.3639585783239454e-05,
"num_tokens": 1585386.0,
"reward": 0.398086816072464,
"reward_std": 0.39999622106552124,
"rewards/humor_reward/mean": 0.398086816072464,
"rewards/humor_reward/std": 0.39999625086784363,
"sampling/importance_sampling_ratio/max": 0.011358072981238365,
"sampling/importance_sampling_ratio/mean": 0.0010091899894177914,
"sampling/importance_sampling_ratio/min": 4.4679931976432385e-10,
"sampling/sampling_logp_difference/max": 3.5639562606811523,
"sampling/sampling_logp_difference/mean": 0.035584691911935806,
"step": 22,
"step_time": 77.29983079498925
},
{
"cispo_clip_ratio": 0.5936666019260883,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4713.3125,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 701.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.1694510877132416,
"epoch": 0.08487084870848709,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00025732198264449835,
"learning_rate": 0.0002,
"loss": 0.00032361538615077734,
"num_tokens": 1664511.0,
"reward": 0.3887137174606323,
"reward_std": 0.35025978088378906,
"rewards/humor_reward/mean": 0.3887137174606323,
"rewards/humor_reward/std": 0.35025978088378906,
"sampling/importance_sampling_ratio/max": 0.2608579397201538,
"sampling/importance_sampling_ratio/mean": 0.016369398683309555,
"sampling/importance_sampling_ratio/min": 5.653824977636113e-11,
"sampling/sampling_logp_difference/max": 1.6544189453125,
"sampling/sampling_logp_difference/mean": 0.03655308857560158,
"step": 23,
"step_time": 79.8481750869978
},
{
"cispo_clip_ratio": 0.46904783695936203,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4739.25,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 2132.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.2212552726268768,
"epoch": 0.08856088560885608,
"frac_reward_zero_std": 0.0,
"grad_norm": 2.1028017727076076e-06,
"learning_rate": 0.0002,
"loss": 5.860923010914121e-06,
"num_tokens": 1744499.0,
"reward": 0.33275866508483887,
"reward_std": 0.3678642511367798,
"rewards/humor_reward/mean": 0.33275866508483887,
"rewards/humor_reward/std": 0.3678642809391022,
"sampling/importance_sampling_ratio/max": 0.0013681778218597174,
"sampling/importance_sampling_ratio/mean": 9.336201765108854e-05,
"sampling/importance_sampling_ratio/min": 2.4985560842516463e-10,
"sampling/sampling_logp_difference/max": 1.9406442642211914,
"sampling/sampling_logp_difference/mean": 0.03511609137058258,
"step": 24,
"step_time": 79.88214839099965
},
{
"cispo_clip_ratio": 0.4411753863096237,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4651.4375,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1004.0,
"completions/min_terminated_length": 0.0,
"entropy": 0.8522009998559952,
"epoch": 0.09225092250922509,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00017764404765330255,
"learning_rate": 0.0002,
"loss": 0.0008231036481447518,
"num_tokens": 1823082.0,
"reward": 0.274623304605484,
"reward_std": 0.3160068690776825,
"rewards/humor_reward/mean": 0.274623304605484,
"rewards/humor_reward/std": 0.3160068988800049,
"sampling/importance_sampling_ratio/max": 0.03566610440611839,
"sampling/importance_sampling_ratio/mean": 0.0037315732333809137,
"sampling/importance_sampling_ratio/min": 4.288159161092153e-09,
"sampling/sampling_logp_difference/max": 2.191567897796631,
"sampling/sampling_logp_difference/mean": 0.027100346982479095,
"step": 25,
"step_time": 65.47632334999798
},
{
"cispo_clip_ratio": 0.3125321865081787,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 5114.5625,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 354.0,
"completions/min_terminated_length": 0.0,
"entropy": 0.8008184731006622,
"epoch": 0.0959409594095941,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00044094581971876323,
"learning_rate": 0.0002,
"loss": -0.0023368855472654104,
"num_tokens": 1909075.0,
"reward": 0.2744894027709961,
"reward_std": 0.3853631615638733,
"rewards/humor_reward/mean": 0.2744894027709961,
"rewards/humor_reward/std": 0.3853631913661957,
"sampling/importance_sampling_ratio/max": 0.27399709820747375,
"sampling/importance_sampling_ratio/mean": 0.02907378226518631,
"sampling/importance_sampling_ratio/min": 1.0831848751280404e-09,
"sampling/sampling_logp_difference/max": 2.079195022583008,
"sampling/sampling_logp_difference/mean": 0.027823781594634056,
"step": 26,
"step_time": 79.29085839301115
},
{
"cispo_clip_ratio": 0.45367857813835144,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 4420.875,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1406.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.1720404550433159,
"epoch": 0.0996309963099631,
"frac_reward_zero_std": 0.0,
"grad_norm": 2.0236677300999872e-05,
"learning_rate": 0.0002,
"loss": -0.0002367804991081357,
"num_tokens": 1983073.0,
"reward": 0.37771666049957275,
"reward_std": 0.3381035327911377,
"rewards/humor_reward/mean": 0.37771666049957275,
"rewards/humor_reward/std": 0.3381035625934601,
"sampling/importance_sampling_ratio/max": 0.01442171260714531,
"sampling/importance_sampling_ratio/mean": 0.0017098486423492432,
"sampling/importance_sampling_ratio/min": 2.375326602077621e-08,
"sampling/sampling_logp_difference/max": 2.334380626678467,
"sampling/sampling_logp_difference/mean": 0.03178582713007927,
"step": 27,
"step_time": 75.20771357801277
},
{
"cispo_clip_ratio": 0.6514521557837725,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 5111.0,
"completions/mean_length": 3562.625,
"completions/mean_terminated_length": 5111.0,
"completions/min_length": 171.0,
"completions/min_terminated_length": 5111.0,
"entropy": 1.2078422084450722,
"epoch": 0.1033210332103321,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.000864825677126646,
"learning_rate": 0.0002,
"loss": 0.0034890188835561275,
"num_tokens": 2044011.0,
"reward": 0.5999776124954224,
"reward_std": 0.378849059343338,
"rewards/humor_reward/mean": 0.5999776124954224,
"rewards/humor_reward/std": 0.3788490891456604,
"sampling/importance_sampling_ratio/max": 0.4461648166179657,
"sampling/importance_sampling_ratio/mean": 0.056391313672065735,
"sampling/importance_sampling_ratio/min": 1.9029215536647826e-08,
"sampling/sampling_logp_difference/max": 2.0776968002319336,
"sampling/sampling_logp_difference/mean": 0.04053337126970291,
"step": 28,
"step_time": 59.718489810984465
},
{
"cispo_clip_ratio": 0.5842036623507738,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3761.1875,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1373.0,
"completions/min_terminated_length": 0.0,
"entropy": 0.9947392120957375,
"epoch": 0.1070110701107011,
"frac_reward_zero_std": 0.0,
"grad_norm": 5.404234002526209e-07,
"learning_rate": 0.0002,
"loss": 3.1678846426075324e-06,
"num_tokens": 2108798.0,
"reward": 0.5179843902587891,
"reward_std": 0.2485807240009308,
"rewards/humor_reward/mean": 0.5179843902587891,
"rewards/humor_reward/std": 0.2485807240009308,
"sampling/importance_sampling_ratio/max": 0.00023896525090094656,
"sampling/importance_sampling_ratio/mean": 1.7564820154802874e-05,
"sampling/importance_sampling_ratio/min": 8.362109155070005e-15,
"sampling/sampling_logp_difference/max": 3.4245944023132324,
"sampling/sampling_logp_difference/mean": 0.04190240800380707,
"step": 29,
"step_time": 92.04002382898761
},
{
"cispo_clip_ratio": 0.481358059681952,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 3548.0,
"completions/mean_length": 3230.25,
"completions/mean_terminated_length": 3548.0,
"completions/min_length": 142.0,
"completions/min_terminated_length": 3548.0,
"entropy": 0.9884351938962936,
"epoch": 0.11070110701107011,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0007160792592912912,
"learning_rate": 0.0002,
"loss": 0.0030249895062297583,
"num_tokens": 2165090.0,
"reward": 0.32606449723243713,
"reward_std": 0.2554178833961487,
"rewards/humor_reward/mean": 0.32606449723243713,
"rewards/humor_reward/std": 0.2554178833961487,
"sampling/importance_sampling_ratio/max": 0.19502675533294678,
"sampling/importance_sampling_ratio/mean": 0.01895805075764656,
"sampling/importance_sampling_ratio/min": 1.5065569625161146e-10,
"sampling/sampling_logp_difference/max": 2.431838035583496,
"sampling/sampling_logp_difference/mean": 0.04055629298090935,
"step": 30,
"step_time": 83.23436555001535
},
{
"cispo_clip_ratio": 0.6561583578586578,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3374.625,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 247.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.0753171369433403,
"epoch": 0.11439114391143912,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0035273912362754345,
"learning_rate": 0.0002,
"loss": 0.01652080938220024,
"num_tokens": 2223020.0,
"reward": 0.5712062120437622,
"reward_std": 0.3373561203479767,
"rewards/humor_reward/mean": 0.5712062120437622,
"rewards/humor_reward/std": 0.3373561203479767,
"sampling/importance_sampling_ratio/max": 1.9402817487716675,
"sampling/importance_sampling_ratio/mean": 0.13337960839271545,
"sampling/importance_sampling_ratio/min": 1.6416909431882232e-07,
"sampling/sampling_logp_difference/max": 6.734940528869629,
"sampling/sampling_logp_difference/mean": 0.03378872573375702,
"step": 31,
"step_time": 59.33454087101563
},
{
"cispo_clip_ratio": 0.6109768375754356,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3135.3125,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 350.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.203492984175682,
"epoch": 0.11808118081180811,
"frac_reward_zero_std": 0.0,
"grad_norm": 8.204303594538942e-05,
"learning_rate": 0.0002,
"loss": 0.0006842454895377159,
"num_tokens": 2277121.0,
"reward": 0.6346642971038818,
"reward_std": 0.2692297101020813,
"rewards/humor_reward/mean": 0.6346642971038818,
"rewards/humor_reward/std": 0.2692297399044037,
"sampling/importance_sampling_ratio/max": 1.2163432836532593,
"sampling/importance_sampling_ratio/mean": 0.07933147996664047,
"sampling/importance_sampling_ratio/min": 3.8786645006894105e-09,
"sampling/sampling_logp_difference/max": 1.4427013397216797,
"sampling/sampling_logp_difference/mean": 0.03805601969361305,
"step": 32,
"step_time": 56.60236855900439
},
{
"cispo_clip_ratio": 0.6497912313789129,
"completions/clipped_ratio": 0.875,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 4153.0,
"completions/mean_length": 3000.5625,
"completions/mean_terminated_length": 2144.5,
"completions/min_length": 136.0,
"completions/min_terminated_length": 136.0,
"entropy": 1.1894647032022476,
"epoch": 0.12177121771217712,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00014119563275016844,
"learning_rate": 0.0002,
"loss": 0.0002433458430459723,
"num_tokens": 2329962.0,
"reward": 0.5449860095977783,
"reward_std": 0.3282092809677124,
"rewards/humor_reward/mean": 0.5449860095977783,
"rewards/humor_reward/std": 0.3282092809677124,
"sampling/importance_sampling_ratio/max": 0.08825313299894333,
"sampling/importance_sampling_ratio/mean": 0.006083119660615921,
"sampling/importance_sampling_ratio/min": 1.4223322519768544e-09,
"sampling/sampling_logp_difference/max": 2.7407069206237793,
"sampling/sampling_logp_difference/mean": 0.04315001145005226,
"step": 33,
"step_time": 87.83407789799094
},
{
"cispo_clip_ratio": 0.603145282715559,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 5923.0,
"completions/max_terminated_length": 1476.0,
"completions/mean_length": 3187.3125,
"completions/mean_terminated_length": 1476.0,
"completions/min_length": 1006.0,
"completions/min_terminated_length": 1476.0,
"entropy": 1.1579975709319115,
"epoch": 0.12546125461254612,
"frac_reward_zero_std": 0.0,
"grad_norm": 5.3547359129879624e-05,
"learning_rate": 0.0002,
"loss": 0.0002505858719814569,
"num_tokens": 2385343.0,
"reward": 0.5027609467506409,
"reward_std": 0.23998978734016418,
"rewards/humor_reward/mean": 0.5027609467506409,
"rewards/humor_reward/std": 0.23998978734016418,
"sampling/importance_sampling_ratio/max": 0.03983650356531143,
"sampling/importance_sampling_ratio/mean": 0.003191877156496048,
"sampling/importance_sampling_ratio/min": 9.33344068698716e-09,
"sampling/sampling_logp_difference/max": 2.9324636459350586,
"sampling/sampling_logp_difference/mean": 0.04020746424794197,
"step": 34,
"step_time": 55.506949331000214
},
{
"cispo_clip_ratio": 0.6451264955103397,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 5022.0,
"completions/max_terminated_length": 529.0,
"completions/mean_length": 2561.125,
"completions/mean_terminated_length": 529.0,
"completions/min_length": 529.0,
"completions/min_terminated_length": 529.0,
"entropy": 1.1418700069189072,
"epoch": 0.12915129151291513,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0015835068188607693,
"learning_rate": 0.0002,
"loss": -0.004869138356298208,
"num_tokens": 2430257.0,
"reward": 0.5386431217193604,
"reward_std": 0.27275699377059937,
"rewards/humor_reward/mean": 0.5386431217193604,
"rewards/humor_reward/std": 0.27275702357292175,
"sampling/importance_sampling_ratio/max": 0.4542810022830963,
"sampling/importance_sampling_ratio/mean": 0.035323094576597214,
"sampling/importance_sampling_ratio/min": 4.0660263350700276e-11,
"sampling/sampling_logp_difference/max": 17.207799911499023,
"sampling/sampling_logp_difference/mean": 0.038973793387413025,
"step": 35,
"step_time": 69.47620079400076
},
{
"cispo_clip_ratio": 0.6264413706958294,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 4404.0,
"completions/mean_length": 3681.5,
"completions/mean_terminated_length": 4404.0,
"completions/min_length": 524.0,
"completions/min_terminated_length": 4404.0,
"entropy": 1.4374851435422897,
"epoch": 0.13284132841328414,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0009762643603608012,
"learning_rate": 0.0002,
"loss": 0.006543246563524008,
"num_tokens": 2493321.0,
"reward": 0.5082834959030151,
"reward_std": 0.34550100564956665,
"rewards/humor_reward/mean": 0.5082834959030151,
"rewards/humor_reward/std": 0.34550103545188904,
"sampling/importance_sampling_ratio/max": 0.3936307430267334,
"sampling/importance_sampling_ratio/mean": 0.03291841596364975,
"sampling/importance_sampling_ratio/min": 5.0612318780451915e-09,
"sampling/sampling_logp_difference/max": 2.5245909690856934,
"sampling/sampling_logp_difference/mean": 0.03988190367817879,
"step": 36,
"step_time": 59.509835421005846
},
{
"cispo_clip_ratio": 0.6464070416986942,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3666.25,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1304.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.4860893040895462,
"epoch": 0.13653136531365315,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0014742233324795961,
"learning_rate": 0.0002,
"loss": 0.009775211103260517,
"num_tokens": 2555917.0,
"reward": 0.5413703918457031,
"reward_std": 0.2888968586921692,
"rewards/humor_reward/mean": 0.5413703918457031,
"rewards/humor_reward/std": 0.2888968884944916,
"sampling/importance_sampling_ratio/max": 0.5927172899246216,
"sampling/importance_sampling_ratio/mean": 0.03740416467189789,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.5622243881225586,
"sampling/sampling_logp_difference/mean": 0.03937282785773277,
"step": 37,
"step_time": 84.5253242639883
},
{
"cispo_clip_ratio": 0.5759155452251434,
"completions/clipped_ratio": 0.875,
"completions/max_length": 5372.0,
"completions/max_terminated_length": 3480.0,
"completions/mean_length": 2918.75,
"completions/mean_terminated_length": 2408.0,
"completions/min_length": 964.0,
"completions/min_terminated_length": 1336.0,
"entropy": 1.4434982985258102,
"epoch": 0.14022140221402213,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.001353645813651383,
"learning_rate": 0.0002,
"loss": -0.005609482992440462,
"num_tokens": 2606553.0,
"reward": 0.27517157793045044,
"reward_std": 0.17297060787677765,
"rewards/humor_reward/mean": 0.27517157793045044,
"rewards/humor_reward/std": 0.17297060787677765,
"sampling/importance_sampling_ratio/max": 0.8227623105049133,
"sampling/importance_sampling_ratio/mean": 0.08773483335971832,
"sampling/importance_sampling_ratio/min": 3.3746889016583737e-07,
"sampling/sampling_logp_difference/max": 1.4419441223144531,
"sampling/sampling_logp_difference/mean": 0.04054056107997894,
"step": 38,
"step_time": 70.05384999897797
},
{
"cispo_clip_ratio": 0.47585512325167656,
"completions/clipped_ratio": 1.0,
"completions/max_length": 3937.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 2637.875,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 631.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.9437123090028763,
"epoch": 0.14391143911439114,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0001363922783639282,
"learning_rate": 0.0002,
"loss": 0.001440045889467001,
"num_tokens": 2652695.0,
"reward": 0.5775443911552429,
"reward_std": 0.1908194124698639,
"rewards/humor_reward/mean": 0.5775443911552429,
"rewards/humor_reward/std": 0.1908194124698639,
"sampling/importance_sampling_ratio/max": 0.04935026168823242,
"sampling/importance_sampling_ratio/mean": 0.003360162954777479,
"sampling/importance_sampling_ratio/min": 9.852194615689314e-09,
"sampling/sampling_logp_difference/max": 2.8174242973327637,
"sampling/sampling_logp_difference/mean": 0.04766388610005379,
"step": 39,
"step_time": 65.56160380599613
},
{
"cispo_clip_ratio": 0.5537771657109261,
"completions/clipped_ratio": 0.875,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 3334.0,
"completions/mean_length": 3318.3125,
"completions/mean_terminated_length": 2593.5,
"completions/min_length": 1005.0,
"completions/min_terminated_length": 1853.0,
"entropy": 1.6230905801057816,
"epoch": 0.14760147601476015,
"frac_reward_zero_std": 0.0,
"grad_norm": 8.798116323305294e-05,
"learning_rate": 0.0002,
"loss": 0.0004228673642501235,
"num_tokens": 2710396.0,
"reward": 0.3043411374092102,
"reward_std": 0.29077214002609253,
"rewards/humor_reward/mean": 0.3043411374092102,
"rewards/humor_reward/std": 0.29077211022377014,
"sampling/importance_sampling_ratio/max": 0.019216708838939667,
"sampling/importance_sampling_ratio/mean": 0.0028733701910823584,
"sampling/importance_sampling_ratio/min": 4.04905886597362e-09,
"sampling/sampling_logp_difference/max": 4.360086441040039,
"sampling/sampling_logp_difference/mean": 0.04630262777209282,
"step": 40,
"step_time": 82.58221219903498
},
{
"cispo_clip_ratio": 0.7186006270349026,
"completions/clipped_ratio": 1.0,
"completions/max_length": 5217.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3383.0,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 706.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.9114204794168472,
"epoch": 0.15129151291512916,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.005707095842808485,
"learning_rate": 0.0002,
"loss": 0.07725385576486588,
"num_tokens": 2768684.0,
"reward": 0.6538974642753601,
"reward_std": 0.22179268300533295,
"rewards/humor_reward/mean": 0.6538974642753601,
"rewards/humor_reward/std": 0.22179268300533295,
"sampling/importance_sampling_ratio/max": 2.3075239658355713,
"sampling/importance_sampling_ratio/mean": 0.24276912212371826,
"sampling/importance_sampling_ratio/min": 2.1112403203460417e-07,
"sampling/sampling_logp_difference/max": 1.9947752952575684,
"sampling/sampling_logp_difference/mean": 0.04460817202925682,
"step": 41,
"step_time": 50.73555830302939
},
{
"cispo_clip_ratio": 0.4581793490797281,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3443.75,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 823.0,
"completions/min_terminated_length": 0.0,
"entropy": 1.684003859758377,
"epoch": 0.15498154981549817,
"frac_reward_zero_std": 0.0,
"grad_norm": 6.124524952610955e-05,
"learning_rate": 0.0002,
"loss": -0.0007328057545237243,
"num_tokens": 2828168.0,
"reward": 0.38168156147003174,
"reward_std": 0.1816880702972412,
"rewards/humor_reward/mean": 0.38168156147003174,
"rewards/humor_reward/std": 0.1816880702972412,
"sampling/importance_sampling_ratio/max": 0.7620577812194824,
"sampling/importance_sampling_ratio/mean": 0.052507393062114716,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 1.3952629566192627,
"sampling/sampling_logp_difference/mean": 0.044263970106840134,
"step": 42,
"step_time": 86.9609940169612
},
{
"cispo_clip_ratio": 0.47743209451436996,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 957.0,
"completions/mean_length": 3697.8125,
"completions/mean_terminated_length": 957.0,
"completions/min_length": 608.0,
"completions/min_terminated_length": 957.0,
"entropy": 1.7134400680661201,
"epoch": 0.15867158671586715,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0007043051300570369,
"learning_rate": 0.0002,
"loss": -0.0017245950875803828,
"num_tokens": 2891493.0,
"reward": 0.22081857919692993,
"reward_std": 0.221486434340477,
"rewards/humor_reward/mean": 0.22081857919692993,
"rewards/humor_reward/std": 0.22148646414279938,
"sampling/importance_sampling_ratio/max": 0.40315449237823486,
"sampling/importance_sampling_ratio/mean": 0.02584739960730076,
"sampling/importance_sampling_ratio/min": 2.196902926243638e-07,
"sampling/sampling_logp_difference/max": 1.3882695436477661,
"sampling/sampling_logp_difference/mean": 0.04035360738635063,
"step": 43,
"step_time": 79.38622622596449
},
{
"cispo_clip_ratio": 0.4560951702296734,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 3416.6875,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 1422.0,
"completions/min_terminated_length": 0.0,
"entropy": 2.0115034580230713,
"epoch": 0.16236162361623616,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00011439115041866899,
"learning_rate": 0.0002,
"loss": 0.0011536129750311375,
"num_tokens": 2950096.0,
"reward": 0.44468164443969727,
"reward_std": 0.2576386332511902,
"rewards/humor_reward/mean": 0.44468164443969727,
"rewards/humor_reward/std": 0.2576386630535126,
"sampling/importance_sampling_ratio/max": 0.04263285920023918,
"sampling/importance_sampling_ratio/mean": 0.004383362829685211,
"sampling/importance_sampling_ratio/min": 1.0335932643101842e-07,
"sampling/sampling_logp_difference/max": 2.972001552581787,
"sampling/sampling_logp_difference/mean": 0.04580405354499817,
"step": 44,
"step_time": 83.37110035201476
},
{
"cispo_clip_ratio": 0.5093751782551408,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 2493.1875,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 544.0,
"completions/min_terminated_length": 0.0,
"entropy": 2.2517920583486557,
"epoch": 0.16605166051660517,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0009305228013545275,
"learning_rate": 0.0002,
"loss": -0.008580348454415798,
"num_tokens": 2994147.0,
"reward": 0.36086541414260864,
"reward_std": 0.23784013092517853,
"rewards/humor_reward/mean": 0.36086541414260864,
"rewards/humor_reward/std": 0.23784014582633972,
"sampling/importance_sampling_ratio/max": 0.34510719776153564,
"sampling/importance_sampling_ratio/mean": 0.02897491306066513,
"sampling/importance_sampling_ratio/min": 1.3008460953388423e-13,
"sampling/sampling_logp_difference/max": 4.940759658813477,
"sampling/sampling_logp_difference/mean": 0.05280106142163277,
"step": 45,
"step_time": 81.29779967200011
},
{
"cispo_clip_ratio": 0.4961427040398121,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 1664.0,
"completions/mean_length": 2631.6875,
"completions/mean_terminated_length": 1664.0,
"completions/min_length": 919.0,
"completions/min_terminated_length": 1664.0,
"entropy": 2.523237004876137,
"epoch": 0.16974169741697417,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0004199541872367263,
"learning_rate": 0.0002,
"loss": 0.0025538511108607054,
"num_tokens": 3040190.0,
"reward": 0.3032156825065613,
"reward_std": 0.2560163736343384,
"rewards/humor_reward/mean": 0.3032156825065613,
"rewards/humor_reward/std": 0.2560163736343384,
"sampling/importance_sampling_ratio/max": 0.6374390125274658,
"sampling/importance_sampling_ratio/mean": 0.047728560864925385,
"sampling/importance_sampling_ratio/min": 3.620046413743694e-07,
"sampling/sampling_logp_difference/max": 2.632134437561035,
"sampling/sampling_logp_difference/mean": 0.047268252819776535,
"step": 46,
"step_time": 73.58007367796381
},
{
"cispo_clip_ratio": 0.3577369898557663,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 1734.0,
"completions/mean_length": 2277.4375,
"completions/mean_terminated_length": 1734.0,
"completions/min_length": 899.0,
"completions/min_terminated_length": 1734.0,
"entropy": 2.4675562232732773,
"epoch": 0.17343173431734318,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.011461591348052025,
"learning_rate": 0.0002,
"loss": 0.08860153704881668,
"num_tokens": 3081013.0,
"reward": 0.2814764380455017,
"reward_std": 0.293722927570343,
"rewards/humor_reward/mean": 0.2814764380455017,
"rewards/humor_reward/std": 0.293722927570343,
"sampling/importance_sampling_ratio/max": 1.583497166633606,
"sampling/importance_sampling_ratio/mean": 0.19163595139980316,
"sampling/importance_sampling_ratio/min": 1.2635299029284397e-08,
"sampling/sampling_logp_difference/max": 3.013592481613159,
"sampling/sampling_logp_difference/mean": 0.05257761478424072,
"step": 47,
"step_time": 80.03998614201555
},
{
"cispo_clip_ratio": 0.343157634139061,
"completions/clipped_ratio": 0.875,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 1813.0,
"completions/mean_length": 1572.6875,
"completions/mean_terminated_length": 1556.0,
"completions/min_length": 505.0,
"completions/min_terminated_length": 1299.0,
"entropy": 2.2395776584744453,
"epoch": 0.17712177121771217,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0015310291200876236,
"learning_rate": 0.0002,
"loss": 0.007011768873780966,
"num_tokens": 3110336.0,
"reward": 0.25455841422080994,
"reward_std": 0.21981197595596313,
"rewards/humor_reward/mean": 0.25455841422080994,
"rewards/humor_reward/std": 0.21981199085712433,
"sampling/importance_sampling_ratio/max": 1.104674220085144,
"sampling/importance_sampling_ratio/mean": 0.09266799688339233,
"sampling/importance_sampling_ratio/min": 6.321847649815027e-07,
"sampling/sampling_logp_difference/max": 2.861496925354004,
"sampling/sampling_logp_difference/mean": 0.048282306641340256,
"step": 48,
"step_time": 74.09863769897493
},
{
"cispo_clip_ratio": 0.2400471270084381,
"completions/clipped_ratio": 1.0,
"completions/max_length": 6144.0,
"completions/max_terminated_length": 0.0,
"completions/mean_length": 1609.875,
"completions/mean_terminated_length": 0.0,
"completions/min_length": 105.0,
"completions/min_terminated_length": 0.0,
"entropy": 2.189691785722971,
"epoch": 0.18081180811808117,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.005181934684514999,
"learning_rate": 0.0002,
"loss": 0.004273830913007259,
"num_tokens": 3140478.0,
"reward": 0.2623010277748108,
"reward_std": 0.2546696662902832,
"rewards/humor_reward/mean": 0.2623010277748108,
"rewards/humor_reward/std": 0.2546696662902832,
"sampling/importance_sampling_ratio/max": 1.6369633674621582,
"sampling/importance_sampling_ratio/mean": 0.1263563483953476,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 1.9059357643127441,
"sampling/sampling_logp_difference/mean": 0.043733954429626465,
"step": 49,
"step_time": 75.97332381599699
},
{
"cispo_clip_ratio": 0.43684835731983185,
"completions/clipped_ratio": 0.9375,
"completions/max_length": 2413.0,
"completions/max_terminated_length": 791.0,
"completions/mean_length": 951.875,
"completions/mean_terminated_length": 791.0,
"completions/min_length": 434.0,
"completions/min_terminated_length": 791.0,
"entropy": 2.404914140701294,
"epoch": 0.18450184501845018,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00264533469453454,
"learning_rate": 0.0002,
"loss": 0.007856368087232113,
"num_tokens": 3159868.0,
"reward": 0.24542748928070068,
"reward_std": 0.2106892615556717,
"rewards/humor_reward/mean": 0.24542748928070068,
"rewards/humor_reward/std": 0.21068927645683289,
"sampling/importance_sampling_ratio/max": 0.8078871965408325,
"sampling/importance_sampling_ratio/mean": 0.06913819909095764,
"sampling/importance_sampling_ratio/min": 8.870298984220426e-08,
"sampling/sampling_logp_difference/max": 3.869755744934082,
"sampling/sampling_logp_difference/mean": 0.0640038326382637,
"step": 50,
"step_time": 23.586737716992502
}
],
"logging_steps": 1,
"max_steps": 200,
"num_input_tokens_seen": 3159868,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}
|