Buckets:
| {"examples": 5987, "tokens": 9210112, "readout_from_votes": 3162, "reasoner_examples": 2818, "reasoner_no_correct_chain": 351, "readout_gold_fallback": 7} | |
| Loading weights: 0%| | 0/426 [00:00<?, ?it/s] Loading weights: 85%|████████▍ | 361/426 [00:00<00:00, 3587.39it/s] Loading weights: 100%|██████████| 426/426 [00:00<00:00, 3588.38it/s] | |
| trainable params: 60,948,480 || all params: 4,266,699,776 || trainable%: 1.4285 | |
| [transformers] `causal_conv1d_fn` is falling back to its reference PyTorch implementation because `causal_conv1d` is not installed. This is correct but much slower; install `causal_conv1d` for the optimized kernel. | |
| /workspace/decision-reasoner/train.py:158: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior. | |
| Consider using tensor.detach() first. (Triggered internally at /__w/pytorch/pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:822.) | |
| stats["chain_loss"] += float(loss) | |
| {"probe_tok_per_s": 2110.0069727294785, "fit_tokens": 8728120.773889866, "have_tokens": 9210112} | |
| {"kept_fraction": 0.9476671699421099, "examples_after_drop": 5669} | |
| {"step": 1, "of": 207, "lr": 1.6666666666666667e-05, "grad_norm": 16.353824615478516, "examples": 30, "readout_examples": 14, "reasoner_examples": 16, "step_tokens": 40439, "chain_tokens": 21065, "micro_batches": 3, "tok_per_s": 2929.685804489287, "elapsed_min": 0.23005311091740926, "eta_min": 47.39094084898631, "chain_loss": 0.1679091975092888, "readout_loss": 1.934458957186767, "readout_argmax_matches_target": 0.42857142857142855, "readout_mean_top_prob": 0.5294180842382568, "readout_target_entropy": 0.2086534925869533, "mem_gb": 19.085318656} | |
| {"step": 2, "of": 207, "lr": 3.3333333333333335e-05, "grad_norm": 7.542080402374268, "examples": 8, "readout_examples": 1, "reasoner_examples": 7, "step_tokens": 43052, "chain_tokens": 26243, "micro_batches": 3, "tok_per_s": 2991.560821974237, "elapsed_min": 0.46514737606048584, "eta_min": 47.6776060461998, "chain_loss": 0.22998714393803052, "readout_loss": 0.6888041496276855, "readout_argmax_matches_target": 1.0, "readout_mean_top_prob": 0.5021762847900391, "readout_target_entropy": 0.0, "mem_gb": 19.085318656} | |
| {"step": 3, "of": 207, "lr": 5e-05, "grad_norm": 12.8873872756958, "examples": 61, "readout_examples": 57, "reasoner_examples": 4, "step_tokens": 39601, "chain_tokens": 22006, "micro_batches": 3, "tok_per_s": 1936.670188000899, "elapsed_min": 1.0593096057573954, "eta_min": 72.03305319150289, "chain_loss": 0.35129909589886665, "readout_loss": 0.8425638817338551, "readout_argmax_matches_target": 0.5964912280701754, "readout_mean_top_prob": 0.6768302318819782, "readout_target_entropy": 0.16120485222914763, "mem_gb": 19.085318656} | |
| {"step": 10, "of": 207, "lr": 9.99023153874608e-05, "grad_norm": 6.824309349060059, "examples": 31, "readout_examples": 5, "reasoner_examples": 26, "step_tokens": 46378, "chain_tokens": 26181, "micro_batches": 3, "tok_per_s": 2601.1282333324484, "elapsed_min": 2.768458922704061, "eta_min": 54.53864077727, "chain_loss": 0.1670885752313412, "readout_loss": 0.9845329189673067, "readout_argmax_matches_target": 0.6, "readout_mean_top_prob": 0.6901487231254577, "readout_target_entropy": 0.10656597912311554, "mem_gb": 19.085318656} | |
| {"step": 20, "of": 207, "lr": 9.880774227376726e-05, "grad_norm": 4.599339008331299, "examples": 28, "readout_examples": 9, "reasoner_examples": 19, "step_tokens": 40504, "chain_tokens": 30459, "micro_batches": 3, "tok_per_s": 2807.820395148787, "elapsed_min": 5.093814651171367, "eta_min": 47.627166988452274, "chain_loss": 0.21822420193960793, "readout_loss": 1.1004138191541035, "readout_argmax_matches_target": 0.5555555555555556, "readout_mean_top_prob": 0.5338592761092715, "readout_target_entropy": 0.0, "mem_gb": 19.085318656} | |
| {"step": 30, "of": 207, "lr": 9.652326481535435e-05, "grad_norm": 12.56348991394043, "examples": 22, "readout_examples": 8, "reasoner_examples": 14, "step_tokens": 36999, "chain_tokens": 23412, "micro_batches": 3, "tok_per_s": 2961.0439312334515, "elapsed_min": 7.217820413907369, "eta_min": 42.58514044205348, "chain_loss": 0.18116038452301705, "readout_loss": 1.2184196175076067, "readout_argmax_matches_target": 0.625, "readout_mean_top_prob": 0.7747065685689449, "readout_target_entropy": 0.31010043574497104, "mem_gb": 19.085318656} | |
| {"step": 40, "of": 207, "lr": 9.310457724282034e-05, "grad_norm": 2.214334011077881, "examples": 122, "readout_examples": 110, "reasoner_examples": 12, "step_tokens": 39960, "chain_tokens": 14950, "micro_batches": 3, "tok_per_s": 2968.213244752267, "elapsed_min": 9.658481709162395, "eta_min": 40.324161135752995, "chain_loss": 0.17671912411848703, "readout_loss": 0.6666781993836842, "readout_argmax_matches_target": 0.6909090909090909, "readout_mean_top_prob": 0.732171431183815, "readout_target_entropy": 0.22204227608807825, "mem_gb": 19.085318656} | |
| {"step": 50, "of": 207, "lr": 8.863502516910058e-05, "grad_norm": 0.2128523737192154, "examples": 14, "readout_examples": 0, "reasoner_examples": 14, "step_tokens": 42258, "chain_tokens": 33152, "micro_batches": 3, "tok_per_s": 2991.9116578971434, "elapsed_min": 11.975854937235514, "eta_min": 37.60418450291952, "chain_loss": 0.15201139077544212, "readout_loss": null, "readout_argmax_matches_target": null, "readout_mean_top_prob": null, "readout_target_entropy": null, "mem_gb": 19.085318656} | |
| {"step": 60, "of": 207, "lr": 8.322357367194109e-05, "grad_norm": 8.037028312683105, "examples": 30, "readout_examples": 12, "reasoner_examples": 18, "step_tokens": 44774, "chain_tokens": 30106, "micro_batches": 3, "tok_per_s": 3034.553312825412, "elapsed_min": 14.053682899475097, "eta_min": 34.431523103713985, "chain_loss": 0.22104241698980331, "readout_loss": 1.124840331884722, "readout_argmax_matches_target": 0.4166666666666667, "readout_mean_top_prob": 0.5562434966365496, "readout_target_entropy": 0.07488711923360825, "mem_gb": 19.085318656} | |
| {"step": 70, "of": 207, "lr": 7.700215078873379e-05, "grad_norm": 6.59724235534668, "examples": 24, "readout_examples": 8, "reasoner_examples": 16, "step_tokens": 42708, "chain_tokens": 30189, "micro_batches": 3, "tok_per_s": 3048.597837583298, "elapsed_min": 16.3681587378184, "eta_min": 32.03482495830172, "chain_loss": 0.1460376912727952, "readout_loss": 0.8100452562794089, "readout_argmax_matches_target": 0.625, "readout_mean_top_prob": 0.6200514361262321, "readout_target_entropy": 0.17352180555462837, "mem_gb": 19.085318656} | |
| {"step": 80, "of": 207, "lr": 7.012243118776269e-05, "grad_norm": 4.699362277984619, "examples": 13, "readout_examples": 1, "reasoner_examples": 12, "step_tokens": 42375, "chain_tokens": 31147, "micro_batches": 3, "tok_per_s": 3029.414900353085, "elapsed_min": 18.69640459616979, "eta_min": 29.680542296419542, "chain_loss": 0.20653490660091242, "readout_loss": 0.7964814901351929, "readout_argmax_matches_target": 1.0, "readout_mean_top_prob": 0.4509127140045166, "readout_target_entropy": 0.0, "mem_gb": 19.085318656} | |
| {"step": 90, "of": 207, "lr": 6.275213842808383e-05, "grad_norm": 7.471736907958984, "examples": 15, "readout_examples": 3, "reasoner_examples": 12, "step_tokens": 44852, "chain_tokens": 27630, "micro_batches": 3, "tok_per_s": 3057.3651444803836, "elapsed_min": 20.847308162848154, "eta_min": 27.101500611702598, "chain_loss": 0.20712771515051523, "readout_loss": 1.5815004110336304, "readout_argmax_matches_target": 0.3333333333333333, "readout_mean_top_prob": 0.42134852210680646, "readout_target_entropy": 0.12559005618095398, "mem_gb": 19.085318656} | |
| {"step": 100, "of": 207, "lr": 5.5070955956790594e-05, "grad_norm": 0.28775501251220703, "examples": 15, "readout_examples": 0, "reasoner_examples": 15, "step_tokens": 45791, "chain_tokens": 33489, "micro_batches": 3, "tok_per_s": 3079.1171606366233, "elapsed_min": 23.077578071753184, "eta_min": 24.693008536775906, "chain_loss": 0.18781028439601263, "readout_loss": null, "readout_argmax_matches_target": null, "readout_mean_top_prob": null, "readout_target_entropy": null, "mem_gb": 19.085318656} | |
| {"step": 110, "of": 207, "lr": 4.726614653117071e-05, "grad_norm": 2.901421070098877, "examples": 64, "readout_examples": 60, "reasoner_examples": 4, "step_tokens": 40137, "chain_tokens": 21425, "micro_batches": 3, "tok_per_s": 3093.3938058169592, "elapsed_min": 25.262571867307027, "eta_min": 22.276995192079834, "chain_loss": 0.2232692390680313, "readout_loss": 0.7240575540345162, "readout_argmax_matches_target": 0.6666666666666666, "readout_mean_top_prob": 0.7004167820016544, "readout_target_entropy": 0.17394846379756929, "mem_gb": 19.085318656} | |
| {"step": 120, "of": 207, "lr": 3.952798686169279e-05, "grad_norm": 0.2123325616121292, "examples": 14, "readout_examples": 0, "reasoner_examples": 14, "step_tokens": 43544, "chain_tokens": 33440, "micro_batches": 3, "tok_per_s": 3097.7480998700826, "elapsed_min": 27.570705314477284, "eta_min": 19.988761352996033, "chain_loss": 0.18893495627811976, "readout_loss": null, "readout_argmax_matches_target": null, "readout_mean_top_prob": null, "readout_target_entropy": null, "mem_gb": 19.085318656} | |
| {"step": 130, "of": 207, "lr": 3.2045128776568784e-05, "grad_norm": 12.950605392456055, "examples": 71, "readout_examples": 65, "reasoner_examples": 6, "step_tokens": 39441, "chain_tokens": 20833, "micro_batches": 3, "tok_per_s": 3107.8540108273446, "elapsed_min": 29.71120361884435, "eta_min": 17.598174451161654, "chain_loss": 0.22970323637127876, "readout_loss": 0.7027810201084671, "readout_argmax_matches_target": 0.6615384615384615, "readout_mean_top_prob": 0.6789758677666003, "readout_target_entropy": 0.18075014352798463, "mem_gb": 19.085318656} | |
| {"step": 140, "of": 207, "lr": 2.500000000000001e-05, "grad_norm": 1.572521686553955, "examples": 19, "readout_examples": 5, "reasoner_examples": 14, "step_tokens": 45279, "chain_tokens": 30104, "micro_batches": 3, "tok_per_s": 3116.8414919136612, "elapsed_min": 31.818113601207735, "eta_min": 15.227240080577985, "chain_loss": 0.22058590341891562, "readout_loss": 0.591015362739563, "readout_argmax_matches_target": 0.8, "readout_mean_top_prob": 0.6042369425296783, "readout_target_entropy": 0.0, "mem_gb": 19.085318656} | |
| {"step": 150, "of": 207, "lr": 1.8564356670455767e-05, "grad_norm": 4.612676620483398, "examples": 15, "readout_examples": 2, "reasoner_examples": 13, "step_tokens": 46038, "chain_tokens": 31670, "micro_batches": 3, "tok_per_s": 3123.031223932227, "elapsed_min": 34.00748900175095, "eta_min": 12.92284582066536, "chain_loss": 0.17214029081738913, "readout_loss": 2.996444523334503, "readout_argmax_matches_target": 0.0, "readout_mean_top_prob": 0.713391900062561, "readout_target_entropy": 0.0, "mem_gb": 19.085318656} | |
| {"step": 160, "of": 207, "lr": 1.289509602599996e-05, "grad_norm": 0.22141993045806885, "examples": 8, "readout_examples": 0, "reasoner_examples": 8, "step_tokens": 37780, "chain_tokens": 33914, "micro_batches": 3, "tok_per_s": 3139.090986909638, "elapsed_min": 36.10835126241048, "eta_min": 10.606828183333079, "chain_loss": 0.24487851280719042, "readout_loss": null, "readout_argmax_matches_target": null, "readout_mean_top_prob": null, "readout_target_entropy": null, "mem_gb": 19.085318656} | |
| {"step": 170, "of": 207, "lr": 8.130431340959981e-06, "grad_norm": 0.2147686630487442, "examples": 10, "readout_examples": 0, "reasoner_examples": 10, "step_tokens": 36655, "chain_tokens": 27672, "micro_batches": 3, "tok_per_s": 3145.1973456729534, "elapsed_min": 38.23120463291804, "eta_min": 8.320909243635104, "chain_loss": 0.170370851829648, "readout_loss": null, "readout_argmax_matches_target": null, "readout_mean_top_prob": null, "readout_target_entropy": null, "mem_gb": 19.085318656} | |
| {"step": 180, "of": 207, "lr": 4.386522366755169e-06, "grad_norm": 4.399493217468262, "examples": 22, "readout_examples": 9, "reasoner_examples": 13, "step_tokens": 46612, "chain_tokens": 32152, "micro_batches": 4, "tok_per_s": 3117.725601520395, "elapsed_min": 40.83136756420136, "eta_min": 6.124705134630203, "chain_loss": 0.19615883895984063, "readout_loss": 0.499316999481784, "readout_argmax_matches_target": 1.0, "readout_mean_top_prob": 0.6652993592951033, "readout_target_entropy": 0.15321971227725348, "mem_gb": 19.782436352} | |
| {"step": 190, "of": 207, "lr": 1.7546434247760146e-06, "grad_norm": 0.7283493280410767, "examples": 17, "readout_examples": 2, "reasoner_examples": 15, "step_tokens": 44357, "chain_tokens": 28694, "micro_batches": 3, "tok_per_s": 3129.445585387388, "elapsed_min": 42.86470121939977, "eta_min": 3.835262740683137, "chain_loss": 0.15698070228099822, "readout_loss": 0.5103972256183624, "readout_argmax_matches_target": 1.0, "readout_mean_top_prob": 0.8205999135971069, "readout_target_entropy": 0.18838508427143097, "mem_gb": 19.782436352} | |
| {"step": 200, "of": 207, "lr": 2.9895819156119943e-07, "grad_norm": 1.8945939540863037, "examples": 19, "readout_examples": 2, "reasoner_examples": 17, "step_tokens": 43523, "chain_tokens": 28779, "micro_batches": 3, "tok_per_s": 3134.7371156418453, "elapsed_min": 45.05890333255132, "eta_min": 1.5770616166392961, "chain_loss": 0.17900991264511557, "readout_loss": 0.46608100086450577, "readout_argmax_matches_target": 1.0, "readout_mean_top_prob": 0.6638901978731155, "readout_target_entropy": 0.0, "mem_gb": 19.782436352} | |
| {"mode": "joint", "alpha": 0.5, "lr": 0.0001, "rank": 32, "lora_alpha": 64, "seed": 17, "examples": 5669, "steps_done": 207, "steps_planned": 207, "stopped_at_deadline": false, "train_tokens": 8743467, "train_seconds": 2787.2046253681183, "tok_per_s": 3137.0021850639014, "counts": {"readout_from_votes": 3162, "reasoner_examples": 2818, "reasoner_no_correct_chain": 351, "readout_gold_fallback": 7, "dropped_for_time": 318}, "reasoner_examples": 2673, "readout_examples": 2996} | |
Xet Storage Details
- Size:
- 13.8 kB
- Xet hash:
- 9b550d728f1e9a387c5c89c1d2a71cd89fcfa20085218c853977c311a73b0e51
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.