Training in progress, step 10000, checkpoint
Browse files
last-checkpoint/model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 223144592
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7866bebbaf9b8e517a84648e60803297129c3fd969715fbdeaae9b2db38d25b6
|
| 3 |
size 223144592
|
last-checkpoint/optimizer.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 440235130
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:56b99d87026a8ade6094641933fad0d4a3c0915eefa6662c667db9b66f557d4e
|
| 3 |
size 440235130
|
last-checkpoint/rng_state.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 14244
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6b83c8e2618548ad0a6cb55d5f9e419ee82c921eaf0cb178e6e4f4700b370bc6
|
| 3 |
size 14244
|
last-checkpoint/scheduler.pt
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 1064
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f9b5c0812245fd57418b0ad79ee0dda344f24eb5c34a21966b50dcec7dccda08
|
| 3 |
size 1064
|
last-checkpoint/trainer_state.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
{
|
| 2 |
"best_metric": Infinity,
|
| 3 |
"best_model_checkpoint": null,
|
| 4 |
-
"epoch":
|
| 5 |
"eval_steps": 1000,
|
| 6 |
-
"global_step":
|
| 7 |
"is_hyper_param_search": false,
|
| 8 |
"is_local_process_zero": true,
|
| 9 |
"is_world_process_zero": true,
|
|
@@ -2608,6 +2608,295 @@
|
|
| 2608 |
"eval_steps_per_second": 1.264,
|
| 2609 |
"eval_wer": Infinity,
|
| 2610 |
"step": 9000
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2611 |
}
|
| 2612 |
],
|
| 2613 |
"logging_steps": 25,
|
|
@@ -2622,12 +2911,12 @@
|
|
| 2622 |
"should_evaluate": false,
|
| 2623 |
"should_log": false,
|
| 2624 |
"should_save": true,
|
| 2625 |
-
"should_training_stop":
|
| 2626 |
},
|
| 2627 |
"attributes": {}
|
| 2628 |
}
|
| 2629 |
},
|
| 2630 |
-
"total_flos": 1.
|
| 2631 |
"train_batch_size": 12,
|
| 2632 |
"trial_name": null,
|
| 2633 |
"trial_params": null
|
|
|
|
| 1 |
{
|
| 2 |
"best_metric": Infinity,
|
| 3 |
"best_model_checkpoint": null,
|
| 4 |
+
"epoch": 16.50206440957886,
|
| 5 |
"eval_steps": 1000,
|
| 6 |
+
"global_step": 10000,
|
| 7 |
"is_hyper_param_search": false,
|
| 8 |
"is_local_process_zero": true,
|
| 9 |
"is_world_process_zero": true,
|
|
|
|
| 2608 |
"eval_steps_per_second": 1.264,
|
| 2609 |
"eval_wer": Infinity,
|
| 2610 |
"step": 9000
|
| 2611 |
+
},
|
| 2612 |
+
{
|
| 2613 |
+
"epoch": 14.893476465730801,
|
| 2614 |
+
"grad_norm": 4.103869915008545,
|
| 2615 |
+
"learning_rate": 2.056842105263158e-06,
|
| 2616 |
+
"loss": 0.1855,
|
| 2617 |
+
"step": 9025
|
| 2618 |
+
},
|
| 2619 |
+
{
|
| 2620 |
+
"epoch": 14.93476465730801,
|
| 2621 |
+
"grad_norm": 4.589334487915039,
|
| 2622 |
+
"learning_rate": 2.0042105263157896e-06,
|
| 2623 |
+
"loss": 0.1723,
|
| 2624 |
+
"step": 9050
|
| 2625 |
+
},
|
| 2626 |
+
{
|
| 2627 |
+
"epoch": 14.97605284888522,
|
| 2628 |
+
"grad_norm": 5.06859016418457,
|
| 2629 |
+
"learning_rate": 1.9515789473684213e-06,
|
| 2630 |
+
"loss": 0.1568,
|
| 2631 |
+
"step": 9075
|
| 2632 |
+
},
|
| 2633 |
+
{
|
| 2634 |
+
"epoch": 15.016515276630884,
|
| 2635 |
+
"grad_norm": 3.8599202632904053,
|
| 2636 |
+
"learning_rate": 1.8989473684210527e-06,
|
| 2637 |
+
"loss": 0.1802,
|
| 2638 |
+
"step": 9100
|
| 2639 |
+
},
|
| 2640 |
+
{
|
| 2641 |
+
"epoch": 15.057803468208093,
|
| 2642 |
+
"grad_norm": 3.51918363571167,
|
| 2643 |
+
"learning_rate": 1.8463157894736843e-06,
|
| 2644 |
+
"loss": 0.163,
|
| 2645 |
+
"step": 9125
|
| 2646 |
+
},
|
| 2647 |
+
{
|
| 2648 |
+
"epoch": 15.099091659785302,
|
| 2649 |
+
"grad_norm": 3.9094550609588623,
|
| 2650 |
+
"learning_rate": 1.7936842105263158e-06,
|
| 2651 |
+
"loss": 0.1508,
|
| 2652 |
+
"step": 9150
|
| 2653 |
+
},
|
| 2654 |
+
{
|
| 2655 |
+
"epoch": 15.14037985136251,
|
| 2656 |
+
"grad_norm": 4.491118907928467,
|
| 2657 |
+
"learning_rate": 1.7410526315789474e-06,
|
| 2658 |
+
"loss": 0.1683,
|
| 2659 |
+
"step": 9175
|
| 2660 |
+
},
|
| 2661 |
+
{
|
| 2662 |
+
"epoch": 15.18166804293972,
|
| 2663 |
+
"grad_norm": 5.513247966766357,
|
| 2664 |
+
"learning_rate": 1.6884210526315792e-06,
|
| 2665 |
+
"loss": 0.1413,
|
| 2666 |
+
"step": 9200
|
| 2667 |
+
},
|
| 2668 |
+
{
|
| 2669 |
+
"epoch": 15.222956234516928,
|
| 2670 |
+
"grad_norm": 4.3536577224731445,
|
| 2671 |
+
"learning_rate": 1.6357894736842107e-06,
|
| 2672 |
+
"loss": 0.1749,
|
| 2673 |
+
"step": 9225
|
| 2674 |
+
},
|
| 2675 |
+
{
|
| 2676 |
+
"epoch": 15.264244426094137,
|
| 2677 |
+
"grad_norm": 3.8934166431427,
|
| 2678 |
+
"learning_rate": 1.5831578947368423e-06,
|
| 2679 |
+
"loss": 0.1727,
|
| 2680 |
+
"step": 9250
|
| 2681 |
+
},
|
| 2682 |
+
{
|
| 2683 |
+
"epoch": 15.305532617671346,
|
| 2684 |
+
"grad_norm": 4.610677242279053,
|
| 2685 |
+
"learning_rate": 1.5305263157894737e-06,
|
| 2686 |
+
"loss": 0.1518,
|
| 2687 |
+
"step": 9275
|
| 2688 |
+
},
|
| 2689 |
+
{
|
| 2690 |
+
"epoch": 15.346820809248555,
|
| 2691 |
+
"grad_norm": 3.230677366256714,
|
| 2692 |
+
"learning_rate": 1.4778947368421054e-06,
|
| 2693 |
+
"loss": 0.1525,
|
| 2694 |
+
"step": 9300
|
| 2695 |
+
},
|
| 2696 |
+
{
|
| 2697 |
+
"epoch": 15.388109000825764,
|
| 2698 |
+
"grad_norm": 3.854262590408325,
|
| 2699 |
+
"learning_rate": 1.425263157894737e-06,
|
| 2700 |
+
"loss": 0.1501,
|
| 2701 |
+
"step": 9325
|
| 2702 |
+
},
|
| 2703 |
+
{
|
| 2704 |
+
"epoch": 15.429397192402973,
|
| 2705 |
+
"grad_norm": 3.96171498298645,
|
| 2706 |
+
"learning_rate": 1.3726315789473684e-06,
|
| 2707 |
+
"loss": 0.1655,
|
| 2708 |
+
"step": 9350
|
| 2709 |
+
},
|
| 2710 |
+
{
|
| 2711 |
+
"epoch": 15.470685383980182,
|
| 2712 |
+
"grad_norm": 3.8165667057037354,
|
| 2713 |
+
"learning_rate": 1.32e-06,
|
| 2714 |
+
"loss": 0.1469,
|
| 2715 |
+
"step": 9375
|
| 2716 |
+
},
|
| 2717 |
+
{
|
| 2718 |
+
"epoch": 15.511973575557391,
|
| 2719 |
+
"grad_norm": 3.6944658756256104,
|
| 2720 |
+
"learning_rate": 1.2673684210526315e-06,
|
| 2721 |
+
"loss": 0.15,
|
| 2722 |
+
"step": 9400
|
| 2723 |
+
},
|
| 2724 |
+
{
|
| 2725 |
+
"epoch": 15.5532617671346,
|
| 2726 |
+
"grad_norm": 3.9097495079040527,
|
| 2727 |
+
"learning_rate": 1.2147368421052633e-06,
|
| 2728 |
+
"loss": 0.1492,
|
| 2729 |
+
"step": 9425
|
| 2730 |
+
},
|
| 2731 |
+
{
|
| 2732 |
+
"epoch": 15.59454995871181,
|
| 2733 |
+
"grad_norm": 5.010981559753418,
|
| 2734 |
+
"learning_rate": 1.1621052631578948e-06,
|
| 2735 |
+
"loss": 0.1551,
|
| 2736 |
+
"step": 9450
|
| 2737 |
+
},
|
| 2738 |
+
{
|
| 2739 |
+
"epoch": 15.635838150289018,
|
| 2740 |
+
"grad_norm": 4.151124000549316,
|
| 2741 |
+
"learning_rate": 1.1094736842105264e-06,
|
| 2742 |
+
"loss": 0.1569,
|
| 2743 |
+
"step": 9475
|
| 2744 |
+
},
|
| 2745 |
+
{
|
| 2746 |
+
"epoch": 15.677126341866225,
|
| 2747 |
+
"grad_norm": 4.455909729003906,
|
| 2748 |
+
"learning_rate": 1.0568421052631578e-06,
|
| 2749 |
+
"loss": 0.1551,
|
| 2750 |
+
"step": 9500
|
| 2751 |
+
},
|
| 2752 |
+
{
|
| 2753 |
+
"epoch": 15.718414533443434,
|
| 2754 |
+
"grad_norm": 4.5624895095825195,
|
| 2755 |
+
"learning_rate": 1.0042105263157897e-06,
|
| 2756 |
+
"loss": 0.1526,
|
| 2757 |
+
"step": 9525
|
| 2758 |
+
},
|
| 2759 |
+
{
|
| 2760 |
+
"epoch": 15.759702725020643,
|
| 2761 |
+
"grad_norm": 3.476386070251465,
|
| 2762 |
+
"learning_rate": 9.515789473684212e-07,
|
| 2763 |
+
"loss": 0.151,
|
| 2764 |
+
"step": 9550
|
| 2765 |
+
},
|
| 2766 |
+
{
|
| 2767 |
+
"epoch": 15.800990916597852,
|
| 2768 |
+
"grad_norm": 3.7594966888427734,
|
| 2769 |
+
"learning_rate": 8.989473684210527e-07,
|
| 2770 |
+
"loss": 0.1614,
|
| 2771 |
+
"step": 9575
|
| 2772 |
+
},
|
| 2773 |
+
{
|
| 2774 |
+
"epoch": 15.842279108175061,
|
| 2775 |
+
"grad_norm": 3.677940607070923,
|
| 2776 |
+
"learning_rate": 8.463157894736843e-07,
|
| 2777 |
+
"loss": 0.1607,
|
| 2778 |
+
"step": 9600
|
| 2779 |
+
},
|
| 2780 |
+
{
|
| 2781 |
+
"epoch": 15.88356729975227,
|
| 2782 |
+
"grad_norm": 2.814276695251465,
|
| 2783 |
+
"learning_rate": 7.936842105263158e-07,
|
| 2784 |
+
"loss": 0.1584,
|
| 2785 |
+
"step": 9625
|
| 2786 |
+
},
|
| 2787 |
+
{
|
| 2788 |
+
"epoch": 15.92485549132948,
|
| 2789 |
+
"grad_norm": 4.551584243774414,
|
| 2790 |
+
"learning_rate": 7.410526315789475e-07,
|
| 2791 |
+
"loss": 0.1746,
|
| 2792 |
+
"step": 9650
|
| 2793 |
+
},
|
| 2794 |
+
{
|
| 2795 |
+
"epoch": 15.966143682906688,
|
| 2796 |
+
"grad_norm": 3.54571533203125,
|
| 2797 |
+
"learning_rate": 6.884210526315791e-07,
|
| 2798 |
+
"loss": 0.1283,
|
| 2799 |
+
"step": 9675
|
| 2800 |
+
},
|
| 2801 |
+
{
|
| 2802 |
+
"epoch": 16.006606110652353,
|
| 2803 |
+
"grad_norm": 3.034546136856079,
|
| 2804 |
+
"learning_rate": 6.357894736842106e-07,
|
| 2805 |
+
"loss": 0.1445,
|
| 2806 |
+
"step": 9700
|
| 2807 |
+
},
|
| 2808 |
+
{
|
| 2809 |
+
"epoch": 16.04789430222956,
|
| 2810 |
+
"grad_norm": 4.239781379699707,
|
| 2811 |
+
"learning_rate": 5.831578947368421e-07,
|
| 2812 |
+
"loss": 0.1526,
|
| 2813 |
+
"step": 9725
|
| 2814 |
+
},
|
| 2815 |
+
{
|
| 2816 |
+
"epoch": 16.08918249380677,
|
| 2817 |
+
"grad_norm": 3.5759620666503906,
|
| 2818 |
+
"learning_rate": 5.305263157894737e-07,
|
| 2819 |
+
"loss": 0.1435,
|
| 2820 |
+
"step": 9750
|
| 2821 |
+
},
|
| 2822 |
+
{
|
| 2823 |
+
"epoch": 16.13047068538398,
|
| 2824 |
+
"grad_norm": 3.807652473449707,
|
| 2825 |
+
"learning_rate": 4.778947368421053e-07,
|
| 2826 |
+
"loss": 0.1616,
|
| 2827 |
+
"step": 9775
|
| 2828 |
+
},
|
| 2829 |
+
{
|
| 2830 |
+
"epoch": 16.17175887696119,
|
| 2831 |
+
"grad_norm": 2.9478118419647217,
|
| 2832 |
+
"learning_rate": 4.2526315789473684e-07,
|
| 2833 |
+
"loss": 0.1344,
|
| 2834 |
+
"step": 9800
|
| 2835 |
+
},
|
| 2836 |
+
{
|
| 2837 |
+
"epoch": 16.213047068538398,
|
| 2838 |
+
"grad_norm": 3.351928472518921,
|
| 2839 |
+
"learning_rate": 3.726315789473685e-07,
|
| 2840 |
+
"loss": 0.1616,
|
| 2841 |
+
"step": 9825
|
| 2842 |
+
},
|
| 2843 |
+
{
|
| 2844 |
+
"epoch": 16.254335260115607,
|
| 2845 |
+
"grad_norm": 4.71775484085083,
|
| 2846 |
+
"learning_rate": 3.2e-07,
|
| 2847 |
+
"loss": 0.138,
|
| 2848 |
+
"step": 9850
|
| 2849 |
+
},
|
| 2850 |
+
{
|
| 2851 |
+
"epoch": 16.295623451692816,
|
| 2852 |
+
"grad_norm": 4.4986138343811035,
|
| 2853 |
+
"learning_rate": 2.6736842105263164e-07,
|
| 2854 |
+
"loss": 0.1592,
|
| 2855 |
+
"step": 9875
|
| 2856 |
+
},
|
| 2857 |
+
{
|
| 2858 |
+
"epoch": 16.336911643270025,
|
| 2859 |
+
"grad_norm": 4.285621643066406,
|
| 2860 |
+
"learning_rate": 2.1473684210526317e-07,
|
| 2861 |
+
"loss": 0.1423,
|
| 2862 |
+
"step": 9900
|
| 2863 |
+
},
|
| 2864 |
+
{
|
| 2865 |
+
"epoch": 16.378199834847234,
|
| 2866 |
+
"grad_norm": 4.092608451843262,
|
| 2867 |
+
"learning_rate": 1.6210526315789476e-07,
|
| 2868 |
+
"loss": 0.1619,
|
| 2869 |
+
"step": 9925
|
| 2870 |
+
},
|
| 2871 |
+
{
|
| 2872 |
+
"epoch": 16.419488026424442,
|
| 2873 |
+
"grad_norm": 4.384539604187012,
|
| 2874 |
+
"learning_rate": 1.0947368421052632e-07,
|
| 2875 |
+
"loss": 0.1606,
|
| 2876 |
+
"step": 9950
|
| 2877 |
+
},
|
| 2878 |
+
{
|
| 2879 |
+
"epoch": 16.46077621800165,
|
| 2880 |
+
"grad_norm": 5.559605121612549,
|
| 2881 |
+
"learning_rate": 5.68421052631579e-08,
|
| 2882 |
+
"loss": 0.164,
|
| 2883 |
+
"step": 9975
|
| 2884 |
+
},
|
| 2885 |
+
{
|
| 2886 |
+
"epoch": 16.50206440957886,
|
| 2887 |
+
"grad_norm": 4.197207450866699,
|
| 2888 |
+
"learning_rate": 4.210526315789474e-09,
|
| 2889 |
+
"loss": 0.1615,
|
| 2890 |
+
"step": 10000
|
| 2891 |
+
},
|
| 2892 |
+
{
|
| 2893 |
+
"epoch": 16.50206440957886,
|
| 2894 |
+
"eval_loss": 1.1515085697174072,
|
| 2895 |
+
"eval_runtime": 702.9423,
|
| 2896 |
+
"eval_samples_per_second": 15.052,
|
| 2897 |
+
"eval_steps_per_second": 1.255,
|
| 2898 |
+
"eval_wer": Infinity,
|
| 2899 |
+
"step": 10000
|
| 2900 |
}
|
| 2901 |
],
|
| 2902 |
"logging_steps": 25,
|
|
|
|
| 2911 |
"should_evaluate": false,
|
| 2912 |
"should_log": false,
|
| 2913 |
"should_save": true,
|
| 2914 |
+
"should_training_stop": true
|
| 2915 |
},
|
| 2916 |
"attributes": {}
|
| 2917 |
}
|
| 2918 |
},
|
| 2919 |
+
"total_flos": 1.949150849531904e+19,
|
| 2920 |
"train_batch_size": 12,
|
| 2921 |
"trial_name": null,
|
| 2922 |
"trial_params": null
|