File size: 32,492 Bytes
4a7322a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 |
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.16,
"eval_steps": 500,
"global_step": 20,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.6875,
"completions/max_length": 1707.0,
"completions/max_terminated_length": 915.0,
"completions/mean_length": 812.703125,
"completions/mean_terminated_length": 386.1499938964844,
"completions/min_length": 88.0,
"completions/min_terminated_length": 88.0,
"entropy": 0.319809939712286,
"epoch": 0.008,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.013934657908976078,
"kl": 0.0,
"learning_rate": 0.0,
"loss": 0.0003,
"num_tokens": 52013.0,
"reward": 0.26906251907348633,
"reward_std": 0.43214160203933716,
"rewards/alfworld_rollout_reward_func/mean": 0.26906251907348633,
"rewards/alfworld_rollout_reward_func/std": 0.48894354701042175,
"rollout/action_tokens_mean": 812.703125,
"rollout/episode_turns_mean": 25.0625,
"rollout/mismatch_mean": 0.015625,
"rollout/mismatch_ratio": 0.015625,
"rollout/truncated_ratio": 0.6875,
"sampling/importance_sampling_ratio/max": 2.401656150817871,
"sampling/importance_sampling_ratio/mean": 0.5413336753845215,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.9913136959075928,
"sampling/sampling_logp_difference/mean": 0.020846465602517128,
"step": 1,
"step_time": 340.2447626780008
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.765625,
"completions/max_length": 1338.0,
"completions/max_terminated_length": 912.0,
"completions/mean_length": 822.90625,
"completions/mean_terminated_length": 481.0000305175781,
"completions/min_length": 240.0,
"completions/min_terminated_length": 240.0,
"entropy": 0.33951753890141845,
"epoch": 0.016,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.005984712392091751,
"kl": 0.0,
"learning_rate": 2.5e-06,
"loss": 0.0064,
"num_tokens": 104679.0,
"reward": 0.1837500035762787,
"reward_std": 0.37211528420448303,
"rewards/alfworld_rollout_reward_func/mean": 0.1837500035762787,
"rewards/alfworld_rollout_reward_func/std": 0.44909608364105225,
"rollout/action_tokens_mean": 822.90625,
"rollout/episode_turns_mean": 26.75,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.765625,
"sampling/importance_sampling_ratio/max": 2.0904664993286133,
"sampling/importance_sampling_ratio/mean": 0.30532318353652954,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.2203328609466553,
"sampling/sampling_logp_difference/mean": 0.021448608487844467,
"step": 2,
"step_time": 376.3160819019986
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.75,
"completions/max_length": 1253.0,
"completions/max_terminated_length": 680.0,
"completions/mean_length": 735.65625,
"completions/mean_terminated_length": 380.5,
"completions/min_length": 145.0,
"completions/min_terminated_length": 145.0,
"entropy": 0.32836992759257555,
"epoch": 0.024,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0066122072748839855,
"kl": 0.001546632844110718,
"learning_rate": 5e-06,
"loss": 0.0026,
"num_tokens": 151761.0,
"reward": 0.21156249940395355,
"reward_std": 0.2755644917488098,
"rewards/alfworld_rollout_reward_func/mean": 0.21156249940395355,
"rewards/alfworld_rollout_reward_func/std": 0.44746631383895874,
"rollout/action_tokens_mean": 735.65625,
"rollout/episode_turns_mean": 26.03125,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.75,
"sampling/importance_sampling_ratio/max": 2.4706838130950928,
"sampling/importance_sampling_ratio/mean": 0.5484369993209839,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 1.5531697273254395,
"sampling/sampling_logp_difference/mean": 0.020412152633070946,
"step": 3,
"step_time": 359.0840177950013
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.546875,
"completions/max_length": 1880.0,
"completions/max_terminated_length": 807.0,
"completions/mean_length": 721.78125,
"completions/mean_terminated_length": 425.72412109375,
"completions/min_length": 112.0,
"completions/min_terminated_length": 112.0,
"entropy": 0.3564013452269137,
"epoch": 0.032,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00901270005851984,
"kl": 0.001597001251866459,
"learning_rate": 4.996791267927632e-06,
"loss": 0.0123,
"num_tokens": 197955.0,
"reward": 0.40953126549720764,
"reward_std": 0.48288512229919434,
"rewards/alfworld_rollout_reward_func/mean": 0.40953126549720764,
"rewards/alfworld_rollout_reward_func/std": 0.5298889875411987,
"rollout/action_tokens_mean": 721.78125,
"rollout/episode_turns_mean": 22.859375,
"rollout/mismatch_mean": 0.015625,
"rollout/mismatch_ratio": 0.015625,
"rollout/truncated_ratio": 0.546875,
"sampling/importance_sampling_ratio/max": 2.607266664505005,
"sampling/importance_sampling_ratio/mean": 0.5041748881340027,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 3.2973906993865967,
"sampling/sampling_logp_difference/mean": 0.022099550813436508,
"step": 4,
"step_time": 324.93751245399835
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.6875,
"completions/max_length": 1772.0,
"completions/max_terminated_length": 1135.0,
"completions/mean_length": 823.984375,
"completions/mean_terminated_length": 511.25,
"completions/min_length": 238.0,
"completions/min_terminated_length": 238.0,
"entropy": 0.37002173252403736,
"epoch": 0.04,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.006255371496081352,
"kl": 0.0018659899324120488,
"learning_rate": 4.987173308479738e-06,
"loss": -0.0049,
"num_tokens": 250690.0,
"reward": 0.2629687488079071,
"reward_std": 0.3148724138736725,
"rewards/alfworld_rollout_reward_func/mean": 0.2629687786102295,
"rewards/alfworld_rollout_reward_func/std": 0.48142871260643005,
"rollout/action_tokens_mean": 823.984375,
"rollout/episode_turns_mean": 25.65625,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.6875,
"sampling/importance_sampling_ratio/max": 2.4376842975616455,
"sampling/importance_sampling_ratio/mean": 0.3966376781463623,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 1.3790063858032227,
"sampling/sampling_logp_difference/mean": 0.022958241403102875,
"step": 5,
"step_time": 428.7077085350011
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.546875,
"completions/max_length": 1754.0,
"completions/max_terminated_length": 1081.0,
"completions/mean_length": 778.265625,
"completions/mean_terminated_length": 513.0689697265625,
"completions/min_length": 89.0,
"completions/min_terminated_length": 89.0,
"entropy": 0.33683515526354313,
"epoch": 0.048,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.014856722205877304,
"kl": 0.0018188257581641665,
"learning_rate": 4.971170810820279e-06,
"loss": 0.0284,
"num_tokens": 300499.0,
"reward": 0.4112499952316284,
"reward_std": 0.4292137622833252,
"rewards/alfworld_rollout_reward_func/mean": 0.4112499952316284,
"rewards/alfworld_rollout_reward_func/std": 0.5246087312698364,
"rollout/action_tokens_mean": 778.265625,
"rollout/episode_turns_mean": 24.0,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.546875,
"sampling/importance_sampling_ratio/max": 2.469376802444458,
"sampling/importance_sampling_ratio/mean": 0.41494274139404297,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.5756630897521973,
"sampling/sampling_logp_difference/mean": 0.020635541528463364,
"step": 6,
"step_time": 329.3498830619992
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.765625,
"completions/max_length": 1750.0,
"completions/max_terminated_length": 882.0,
"completions/mean_length": 834.359375,
"completions/mean_terminated_length": 538.4000244140625,
"completions/min_length": 142.0,
"completions/min_terminated_length": 142.0,
"entropy": 0.37804630119353533,
"epoch": 0.056,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.011092925444245338,
"kl": 0.0017349459521938115,
"learning_rate": 4.948824853131237e-06,
"loss": 0.0135,
"num_tokens": 353898.0,
"reward": 0.1875,
"reward_std": 0.41170334815979004,
"rewards/alfworld_rollout_reward_func/mean": 0.1875,
"rewards/alfworld_rollout_reward_func/std": 0.4403570294380188,
"rollout/action_tokens_mean": 834.359375,
"rollout/episode_turns_mean": 27.0625,
"rollout/mismatch_mean": 0.015625,
"rollout/mismatch_ratio": 0.015625,
"rollout/truncated_ratio": 0.765625,
"sampling/importance_sampling_ratio/max": 2.2279603481292725,
"sampling/importance_sampling_ratio/mean": 0.46820664405822754,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 3.733177900314331,
"sampling/sampling_logp_difference/mean": 0.022584954276680946,
"step": 7,
"step_time": 372.10799923799914
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.765625,
"completions/max_length": 1682.0,
"completions/max_terminated_length": 697.0,
"completions/mean_length": 786.5625,
"completions/mean_terminated_length": 380.8000183105469,
"completions/min_length": 180.0,
"completions/min_terminated_length": 180.0,
"entropy": 0.32334376219660044,
"epoch": 0.064,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.010468119755387306,
"kl": 0.0017176041819766397,
"learning_rate": 4.920192797165511e-06,
"loss": -0.0059,
"num_tokens": 404238.0,
"reward": 0.1939062476158142,
"reward_std": 0.31658345460891724,
"rewards/alfworld_rollout_reward_func/mean": 0.1939062476158142,
"rewards/alfworld_rollout_reward_func/std": 0.44521304965019226,
"rollout/action_tokens_mean": 786.5625,
"rollout/episode_turns_mean": 25.984375,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.765625,
"sampling/importance_sampling_ratio/max": 2.8437609672546387,
"sampling/importance_sampling_ratio/mean": 0.6038376092910767,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.8486413955688477,
"sampling/sampling_logp_difference/mean": 0.0203598253428936,
"step": 8,
"step_time": 434.73393120799847
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.75,
"completions/max_length": 1492.0,
"completions/max_terminated_length": 1027.0,
"completions/mean_length": 818.34375,
"completions/mean_terminated_length": 474.0625,
"completions/min_length": 192.0,
"completions/min_terminated_length": 192.0,
"entropy": 0.32139473129063845,
"epoch": 0.072,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.006268487311899662,
"kl": 0.0026504553206905257,
"learning_rate": 4.8853481410001225e-06,
"loss": -0.0089,
"num_tokens": 456612.0,
"reward": 0.19500000774860382,
"reward_std": 0.29811495542526245,
"rewards/alfworld_rollout_reward_func/mean": 0.19500000774860382,
"rewards/alfworld_rollout_reward_func/std": 0.46079298853874207,
"rollout/action_tokens_mean": 818.34375,
"rollout/episode_turns_mean": 26.109375,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.75,
"sampling/importance_sampling_ratio/max": 2.6596250534057617,
"sampling/importance_sampling_ratio/mean": 0.42990007996559143,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 3.3629698753356934,
"sampling/sampling_logp_difference/mean": 0.019470183178782463,
"step": 9,
"step_time": 399.7356371579963
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.59375,
"completions/max_length": 1381.0,
"completions/max_terminated_length": 1015.0,
"completions/mean_length": 710.296875,
"completions/mean_terminated_length": 485.5384826660156,
"completions/min_length": 193.0,
"completions/min_terminated_length": 193.0,
"entropy": 0.34738565888255835,
"epoch": 0.08,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.010129230096936226,
"kl": 0.0015055066687637009,
"learning_rate": 4.844380330367701e-06,
"loss": 0.0042,
"num_tokens": 502071.0,
"reward": 0.3684375286102295,
"reward_std": 0.3336261808872223,
"rewards/alfworld_rollout_reward_func/mean": 0.3684375286102295,
"rewards/alfworld_rollout_reward_func/std": 0.5042713284492493,
"rollout/action_tokens_mean": 710.296875,
"rollout/episode_turns_mean": 24.859375,
"rollout/mismatch_mean": 0.015625,
"rollout/mismatch_ratio": 0.015625,
"rollout/truncated_ratio": 0.59375,
"sampling/importance_sampling_ratio/max": 2.386782646179199,
"sampling/importance_sampling_ratio/mean": 0.5169863104820251,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 3.2723636627197266,
"sampling/sampling_logp_difference/mean": 0.022041790187358856,
"step": 10,
"step_time": 394.5470729360013
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.859375,
"completions/max_length": 1352.0,
"completions/max_terminated_length": 782.0,
"completions/mean_length": 881.84375,
"completions/mean_terminated_length": 523.4444580078125,
"completions/min_length": 224.0,
"completions/min_terminated_length": 224.0,
"entropy": 0.3616171991452575,
"epoch": 0.088,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00803722906857729,
"kl": 0.0017023698201228399,
"learning_rate": 4.797394529050577e-06,
"loss": 0.0111,
"num_tokens": 558509.0,
"reward": 0.08921875059604645,
"reward_std": 0.2777005434036255,
"rewards/alfworld_rollout_reward_func/mean": 0.08921875059604645,
"rewards/alfworld_rollout_reward_func/std": 0.36485835909843445,
"rollout/action_tokens_mean": 881.84375,
"rollout/episode_turns_mean": 28.40625,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.859375,
"sampling/importance_sampling_ratio/max": 2.7719504833221436,
"sampling/importance_sampling_ratio/mean": 0.4752790629863739,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 3.038581132888794,
"sampling/sampling_logp_difference/mean": 0.02142297849059105,
"step": 11,
"step_time": 396.1487058620023
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.671875,
"completions/max_length": 2292.0,
"completions/max_terminated_length": 901.0,
"completions/mean_length": 860.28125,
"completions/mean_terminated_length": 460.76190185546875,
"completions/min_length": 116.0,
"completions/min_terminated_length": 116.0,
"entropy": 0.35516145126894116,
"epoch": 0.096,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.011235510930418968,
"kl": 0.0016268756971840048,
"learning_rate": 4.744511348926855e-06,
"loss": 0.0048,
"num_tokens": 613567.0,
"reward": 0.26890623569488525,
"reward_std": 0.41516801714897156,
"rewards/alfworld_rollout_reward_func/mean": 0.26890626549720764,
"rewards/alfworld_rollout_reward_func/std": 0.4936746656894684,
"rollout/action_tokens_mean": 860.28125,
"rollout/episode_turns_mean": 25.484375,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.671875,
"sampling/importance_sampling_ratio/max": 2.7596683502197266,
"sampling/importance_sampling_ratio/mean": 0.5103189945220947,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.6376895904541016,
"sampling/sampling_logp_difference/mean": 0.02156790718436241,
"step": 12,
"step_time": 421.574981908002
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.8125,
"completions/max_length": 1382.0,
"completions/max_terminated_length": 775.0,
"completions/mean_length": 836.53125,
"completions/mean_terminated_length": 586.4166870117188,
"completions/min_length": 402.0,
"completions/min_terminated_length": 402.0,
"entropy": 0.35410969890654087,
"epoch": 0.104,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.010335148312151432,
"kl": 0.0018077169770549517,
"learning_rate": 4.685866540361456e-06,
"loss": 0.0068,
"num_tokens": 667105.0,
"reward": 0.14281249046325684,
"reward_std": 0.3198840916156769,
"rewards/alfworld_rollout_reward_func/mean": 0.14281249046325684,
"rewards/alfworld_rollout_reward_func/std": 0.3984073996543884,
"rollout/action_tokens_mean": 836.53125,
"rollout/episode_turns_mean": 28.453125,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.8125,
"sampling/importance_sampling_ratio/max": 2.6870784759521484,
"sampling/importance_sampling_ratio/mean": 0.6008636951446533,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.14093017578125,
"sampling/sampling_logp_difference/mean": 0.021617209538817406,
"step": 13,
"step_time": 369.507772967002
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.6875,
"completions/max_length": 1584.0,
"completions/max_terminated_length": 1063.0,
"completions/mean_length": 793.703125,
"completions/mean_terminated_length": 525.4500122070312,
"completions/min_length": 158.0,
"completions/min_terminated_length": 158.0,
"entropy": 0.34002805408090353,
"epoch": 0.112,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.00711849657818675,
"kl": 0.0019163258293701801,
"learning_rate": 4.621610643736878e-06,
"loss": 0.0062,
"num_tokens": 717902.0,
"reward": 0.2626562714576721,
"reward_std": 0.3680095076560974,
"rewards/alfworld_rollout_reward_func/mean": 0.26265624165534973,
"rewards/alfworld_rollout_reward_func/std": 0.489940881729126,
"rollout/action_tokens_mean": 793.703125,
"rollout/episode_turns_mean": 26.28125,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.6875,
"sampling/importance_sampling_ratio/max": 2.815065860748291,
"sampling/importance_sampling_ratio/mean": 0.45154738426208496,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.7529265880584717,
"sampling/sampling_logp_difference/mean": 0.0205212514847517,
"step": 14,
"step_time": 327.6827599140015
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.84375,
"completions/max_length": 1814.0,
"completions/max_terminated_length": 890.0,
"completions/mean_length": 909.109375,
"completions/mean_terminated_length": 528.7999877929688,
"completions/min_length": 242.0,
"completions/min_terminated_length": 242.0,
"entropy": 0.35556588135659695,
"epoch": 0.12,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.010462122038006783,
"kl": 0.0016676104332873365,
"learning_rate": 4.551908603018191e-06,
"loss": -0.0076,
"num_tokens": 776085.0,
"reward": 0.10140626132488251,
"reward_std": 0.36953580379486084,
"rewards/alfworld_rollout_reward_func/mean": 0.10140625387430191,
"rewards/alfworld_rollout_reward_func/std": 0.38891327381134033,
"rollout/action_tokens_mean": 909.109375,
"rollout/episode_turns_mean": 28.0,
"rollout/mismatch_mean": 0.015625,
"rollout/mismatch_ratio": 0.015625,
"rollout/truncated_ratio": 0.84375,
"sampling/importance_sampling_ratio/max": 2.0437681674957275,
"sampling/importance_sampling_ratio/mean": 0.3665631115436554,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 4.642125606536865,
"sampling/sampling_logp_difference/mean": 0.021498428657650948,
"step": 15,
"step_time": 409.45475878100024
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.703125,
"completions/max_length": 2069.0,
"completions/max_terminated_length": 815.0,
"completions/mean_length": 858.484375,
"completions/mean_terminated_length": 363.0,
"completions/min_length": 128.0,
"completions/min_terminated_length": 128.0,
"entropy": 0.3426384939812124,
"epoch": 0.128,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.007739651948213577,
"kl": 0.0015719706807431066,
"learning_rate": 4.476939342344246e-06,
"loss": -0.002,
"num_tokens": 831028.0,
"reward": 0.2515625059604645,
"reward_std": 0.4047408103942871,
"rewards/alfworld_rollout_reward_func/mean": 0.2515625059604645,
"rewards/alfworld_rollout_reward_func/std": 0.48498478531837463,
"rollout/action_tokens_mean": 858.484375,
"rollout/episode_turns_mean": 25.015625,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.703125,
"sampling/importance_sampling_ratio/max": 2.197270631790161,
"sampling/importance_sampling_ratio/mean": 0.3564271330833435,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 1.3733601570129395,
"sampling/sampling_logp_difference/mean": 0.021193569526076317,
"step": 16,
"step_time": 341.3529842470016
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.765625,
"completions/max_length": 1344.0,
"completions/max_terminated_length": 872.0,
"completions/mean_length": 800.890625,
"completions/mean_terminated_length": 410.0000305175781,
"completions/min_length": 199.0,
"completions/min_terminated_length": 199.0,
"entropy": 0.3335589300841093,
"epoch": 0.136,
"frac_reward_zero_std": 0.0,
"grad_norm": 1.4890336990356445,
"kl": 0.8330444995917787,
"learning_rate": 4.396895306731978e-06,
"loss": 0.0005,
"num_tokens": 882285.0,
"reward": 0.18125000596046448,
"reward_std": 0.24295468628406525,
"rewards/alfworld_rollout_reward_func/mean": 0.18124999105930328,
"rewards/alfworld_rollout_reward_func/std": 0.4482647478580475,
"rollout/action_tokens_mean": 800.890625,
"rollout/episode_turns_mean": 26.265625,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.765625,
"sampling/importance_sampling_ratio/max": 2.3350934982299805,
"sampling/importance_sampling_ratio/mean": 0.37133079767227173,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 10.240796089172363,
"sampling/sampling_logp_difference/mean": 0.020599324256181717,
"step": 17,
"step_time": 436.20165634099976
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.890625,
"completions/max_length": 1359.0,
"completions/max_terminated_length": 872.0,
"completions/mean_length": 864.640625,
"completions/mean_terminated_length": 638.7142944335938,
"completions/min_length": 240.0,
"completions/min_terminated_length": 240.0,
"entropy": 0.3253911091014743,
"epoch": 0.144,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.007197697646915913,
"kl": 0.0016915635187615408,
"learning_rate": 4.3119819680728e-06,
"loss": -0.0099,
"num_tokens": 937622.0,
"reward": 0.06968750059604645,
"reward_std": 0.19933855533599854,
"rewards/alfworld_rollout_reward_func/mean": 0.06968750059604645,
"rewards/alfworld_rollout_reward_func/std": 0.3224163353443146,
"rollout/action_tokens_mean": 864.640625,
"rollout/episode_turns_mean": 28.9375,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.890625,
"sampling/importance_sampling_ratio/max": 2.2717995643615723,
"sampling/importance_sampling_ratio/mean": 0.5041627287864685,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.0254077911376953,
"sampling/sampling_logp_difference/mean": 0.019858187064528465,
"step": 18,
"step_time": 437.1889309570033
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.703125,
"completions/max_length": 2201.0,
"completions/max_terminated_length": 1115.0,
"completions/mean_length": 844.5,
"completions/mean_terminated_length": 540.631591796875,
"completions/min_length": 140.0,
"completions/min_terminated_length": 140.0,
"entropy": 0.33306339103728533,
"epoch": 0.152,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.010224551893770695,
"kl": 0.001574573227117071,
"learning_rate": 4.222417297689217e-06,
"loss": 0.0213,
"num_tokens": 991670.0,
"reward": 0.24437499046325684,
"reward_std": 0.42425790429115295,
"rewards/alfworld_rollout_reward_func/mean": 0.24437500536441803,
"rewards/alfworld_rollout_reward_func/std": 0.48038968443870544,
"rollout/action_tokens_mean": 844.5,
"rollout/episode_turns_mean": 26.171875,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.703125,
"sampling/importance_sampling_ratio/max": 2.5866730213165283,
"sampling/importance_sampling_ratio/mean": 0.5376668572425842,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 1.632045865058899,
"sampling/sampling_logp_difference/mean": 0.01941317319869995,
"step": 19,
"step_time": 348.9997736020032
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.75,
"completions/max_length": 1764.0,
"completions/max_terminated_length": 1085.0,
"completions/mean_length": 937.640625,
"completions/mean_terminated_length": 530.6875,
"completions/min_length": 201.0,
"completions/min_terminated_length": 201.0,
"entropy": 0.3477782104164362,
"epoch": 0.16,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.010612000711262226,
"kl": 0.001602184345756541,
"learning_rate": 4.128431206805556e-06,
"loss": 0.0044,
"num_tokens": 1051679.0,
"reward": 0.1901562511920929,
"reward_std": 0.39348262548446655,
"rewards/alfworld_rollout_reward_func/mean": 0.1901562511920929,
"rewards/alfworld_rollout_reward_func/std": 0.4530303478240967,
"rollout/action_tokens_mean": 937.640625,
"rollout/episode_turns_mean": 26.6875,
"rollout/mismatch_mean": 0.0,
"rollout/mismatch_ratio": 0.0,
"rollout/truncated_ratio": 0.75,
"sampling/importance_sampling_ratio/max": 2.5002288818359375,
"sampling/importance_sampling_ratio/mean": 0.4413934350013733,
"sampling/importance_sampling_ratio/min": 0.0,
"sampling/sampling_logp_difference/max": 2.34731388092041,
"sampling/sampling_logp_difference/mean": 0.0198683924973011,
"step": 20,
"step_time": 353.7044870629943
}
],
"logging_steps": 1,
"max_steps": 64,
"num_input_tokens_seen": 1051679,
"num_train_epochs": 1,
"save_steps": 20,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}
|