Archive final stop status and training logs for iclr-debug-generator-corruption-uniform-10-135b-v1 at step 49000
Browse files- iclr-debug-generator-corruption-uniform-10-135b-v1/STOPPED_BY_USER.json +96 -0
- iclr-debug-generator-corruption-uniform-10-135b-v1/generator/HF_UPLOADED_49000.json +7 -0
- iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-12875.log +0 -0
- iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-25750.log +0 -0
- iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-4769.log +0 -0
- iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-51499.log +0 -0
- iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-954.log +186 -0
- iclr-debug-generator-corruption-uniform-10-135b-v1/git-provenance.json +3 -0
- iclr-debug-generator-corruption-uniform-10-135b-v1/status.json +11 -0
iclr-debug-generator-corruption-uniform-10-135b-v1/STOPPED_BY_USER.json
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"stopped_utc": "2026-09-21T04:57:55.852614+00:00",
|
| 3 |
+
"reason": "User requested evaluation and HF backup at step 49000, then termination of the 0.1 corruption run.",
|
| 4 |
+
"retained_step": 49000,
|
| 5 |
+
"positions": 51380224000,
|
| 6 |
+
"training_service_disabled": true,
|
| 7 |
+
"evaluation_complete": true,
|
| 8 |
+
"hf_commit": "5dc54d5c52dc23d9e99c06f1b5d73a02a7bc6479",
|
| 9 |
+
"verified_files": [
|
| 10 |
+
{
|
| 11 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/manifest.json",
|
| 12 |
+
"size": 30374,
|
| 13 |
+
"verified_digest": "d25feb79581751a9a531430a0fa774fc36516f75"
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/config.yaml",
|
| 17 |
+
"size": 4530,
|
| 18 |
+
"verified_digest": "ff607a3ee2855e32582dd7ce61c77e7176f7e1e0"
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/milestone-iter-49000.pt",
|
| 22 |
+
"size": 2565487499,
|
| 23 |
+
"verified_digest": "de5a213fee1345b8482d3306c6df4bc945bc59fffd0b0a0ee247bf96b5ff5dcf"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/exports/step-49000/ncp.pt",
|
| 27 |
+
"size": 427469726,
|
| 28 |
+
"verified_digest": "ef130ed8f5fc6e085f1226d1470758e2b3ab2e1a58a61b4f8cac9fd6cdc85530"
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/exports/step-49000/vqvae.pt",
|
| 32 |
+
"size": 1300864149,
|
| 33 |
+
"verified_digest": "efc0e4c21be1a0c69ce9cd45b92f031791776a2e54c25c9da69205bfc06e64bd"
|
| 34 |
+
},
|
| 35 |
+
{
|
| 36 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/EVAL_DONE.json",
|
| 37 |
+
"size": 97,
|
| 38 |
+
"verified_digest": "6eadee5faa83606fd115284bfa2677a50b30b8e8"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-0.json",
|
| 42 |
+
"size": 145909,
|
| 43 |
+
"verified_digest": "4a96d35065e9b3dd1380869586e38ae988b22c85"
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-0.samples.json",
|
| 47 |
+
"size": 146276,
|
| 48 |
+
"verified_digest": "7fbe5e3c652f08910c45b027ae84a1f7a1a69109"
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-1.json",
|
| 52 |
+
"size": 148715,
|
| 53 |
+
"verified_digest": "60bf0f591c4f08708d174f34c39520b6a7769531"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-1.samples.json",
|
| 57 |
+
"size": 149082,
|
| 58 |
+
"verified_digest": "a8467a0fce17737cbe1cb111d792881eb99e1bce"
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-2.json",
|
| 62 |
+
"size": 148288,
|
| 63 |
+
"verified_digest": "a4a6d925db648d6dc44285166e29ab9ad0e47fa1"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-2.samples.json",
|
| 67 |
+
"size": 148653,
|
| 68 |
+
"verified_digest": "277a4b96fe306d2e53b03f24dd07e07848e9404c"
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-3.json",
|
| 72 |
+
"size": 145123,
|
| 73 |
+
"verified_digest": "bfb90a871456bad58607a4882026498cdd172004"
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-3.samples.json",
|
| 77 |
+
"size": 145492,
|
| 78 |
+
"verified_digest": "f38e4cf13ffcdd9670a2689918808cad1b871afc"
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-4.json",
|
| 82 |
+
"size": 148576,
|
| 83 |
+
"verified_digest": "f09b31e6ebe481223b138b52312899cf0531a86d"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/random-seed-4.samples.json",
|
| 87 |
+
"size": 148942,
|
| 88 |
+
"verified_digest": "621018b8e107748e3e3775513a5344df0d67c72a"
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"path": "iclr-debug-generator-corruption-uniform-10-135b-v1/generator/evaluation/step-49000/summary.json",
|
| 92 |
+
"size": 876,
|
| 93 |
+
"verified_digest": "c0dbd5ccb3f1ad3e33f2dec51b3ce87235e382d6"
|
| 94 |
+
}
|
| 95 |
+
]
|
| 96 |
+
}
|
iclr-debug-generator-corruption-uniform-10-135b-v1/generator/HF_UPLOADED_49000.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"repo": "iskhare/iclr-debug",
|
| 3 |
+
"prefix": "iclr-debug-generator-corruption-uniform-10-135b-v1",
|
| 4 |
+
"commit": "5dc54d5c52dc23d9e99c06f1b5d73a02a7bc6479",
|
| 5 |
+
"step": 49000,
|
| 6 |
+
"evaluation_complete": true
|
| 7 |
+
}
|
iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-12875.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-25750.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-4769.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-51499.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
iclr-debug-generator-corruption-uniform-10-135b-v1/generator/training-to-954.log
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
wandb: Currently logged in as: iskhare (mstok). Use `wandb login --relogin` to force relogin
|
| 2 |
+
VQVAE, Total parameter count: 223.75M
|
| 3 |
+
VQVAE, Non embedding parameter count: 204.34M
|
| 4 |
+
VQVAE, Total parameter count: 223.75M
|
| 5 |
+
VQVAE, Non embedding parameter count: 204.34M
|
| 6 |
+
VQVAE, Total parameter count: 223.75M
|
| 7 |
+
VQVAE, Non embedding parameter count: 204.34M
|
| 8 |
+
VQVAE, Total parameter count: 223.75M
|
| 9 |
+
VQVAE, Non embedding parameter count: 204.34M
|
| 10 |
+
VQVAE, Total parameter count: 223.75M
|
| 11 |
+
VQVAE, Non embedding parameter count: 204.34M
|
| 12 |
+
VQVAE, Total parameter count: 223.75M
|
| 13 |
+
VQVAE, Non embedding parameter count: 204.34M
|
| 14 |
+
VQVAE, Total parameter count: 223.75M
|
| 15 |
+
VQVAE, Non embedding parameter count: 204.34M
|
| 16 |
+
NCP Transformer, Total parameter count: 104.61M
|
| 17 |
+
NCP Transformer, Non embedding parameter count: 104.30M
|
| 18 |
+
NCP Transformer, Total parameter count: 104.61M
|
| 19 |
+
NCP Transformer, Non embedding parameter count: 104.30M
|
| 20 |
+
NCP Transformer, Total parameter count: 104.61M
|
| 21 |
+
NCP Transformer, Non embedding parameter count: 104.30M
|
| 22 |
+
NCP Transformer, Total parameter count: 104.61M
|
| 23 |
+
NCP Transformer, Non embedding parameter count: 104.30M
|
| 24 |
+
NCP Transformer, Total parameter count: 104.61M
|
| 25 |
+
NCP Transformer, Non embedding parameter count: 104.30M
|
| 26 |
+
NCP Transformer, Total parameter count: 104.61M
|
| 27 |
+
NCP Transformer, Non embedding parameter count: 104.30M
|
| 28 |
+
NCP Transformer, Total parameter count: 104.61M
|
| 29 |
+
NCP Transformer, Non embedding parameter count: 104.30M
|
| 30 |
+
wandb: - Waiting for wandb.init()...
|
| 31 |
+
ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
|
| 32 |
+
/home/ishank/mstok-corruption-uniform-10/train_iclr_debug.py:132: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
|
| 33 |
+
ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
|
| 34 |
+
/home/ishank/mstok-corruption-uniform-10/train_iclr_debug.py:132: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
|
| 35 |
+
ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
|
| 36 |
+
/home/ishank/mstok-corruption-uniform-10/train_iclr_debug.py:132: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
|
| 37 |
+
ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
|
| 38 |
+
/home/ishank/mstok-corruption-uniform-10/train_iclr_debug.py:132: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
|
| 39 |
+
ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
|
| 40 |
+
/home/ishank/mstok-corruption-uniform-10/train_iclr_debug.py:132: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
|
| 41 |
+
ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
|
| 42 |
+
/home/ishank/mstok-corruption-uniform-10/train_iclr_debug.py:132: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
|
| 43 |
+
ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
|
| 44 |
+
wandb: Tracking run with wandb version 0.17.9
|
| 45 |
+
wandb: Run data is saved locally in /home/ishank/mstok-corruption-uniform-10/wandb/run-20260919_061426-iclr-debug-generator-corruption-uniform-10-135b-v1-generator
|
| 46 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 47 |
+
wandb: Syncing run iclr-debug-generator-corruption-uniform-10-135b-v1-generator
|
| 48 |
+
wandb: ⭐️ View project at https://wandb.ai/mstok/iclr-debug
|
| 49 |
+
wandb: 🚀 View run at https://wandb.ai/mstok/iclr-debug/runs/iclr-debug-generator-corruption-uniform-10-135b-v1-generator
|
| 50 |
+
VQVAE, Total parameter count: 223.75M
|
| 51 |
+
VQVAE, Non embedding parameter count: 204.34M
|
| 52 |
+
NCP Transformer, Total parameter count: 104.61M
|
| 53 |
+
NCP Transformer, Non embedding parameter count: 104.30M
|
| 54 |
+
/home/ishank/mstok-corruption-uniform-10/train_iclr_debug.py:132: FutureWarning: `broadcast_buffers` is deprecated. Use `forward_sync_buffers` instead. IMPORTANT: unlike `broadcast_buffers=False`, `forward_sync_buffers=False` still syncs buffers at init. If you rely on buffers NOT being synced at init (e.g. rank-local buffers), keep using `broadcast_buffers=False` until `init_sync_buffers` is available.
|
| 55 |
+
ddp = DDP(model, device_ids=[local_rank] if device.type == "cuda" else None,
|
| 56 |
+
{"step": 1, "stop_at": 954, "target": 128747, "training/loss": 9.87092000246048, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.87092000246048, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.005291074414715719, "training/generator_lr": 4.444444444444444e-06, "training/grad_norm": 0.14678044617176056, "training/positions": 1048576, "training/nonpadding_tokens": 1048576, "throughput/steps_per_second": 0.044308706704531756, "throughput/raw_tokens_per_second": 46461.04644141109, "throughput/peak_reserved_gib": 23.5703125}
|
| 57 |
+
{"step": 10, "stop_at": 954, "target": 128747, "training/loss": 9.844226776725716, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.844226776725716, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.007239844620958751, "training/generator_lr": 2.4444444444444445e-05, "training/grad_norm": 0.11820857971906662, "training/positions": 10485760, "training/nonpadding_tokens": 10485517, "throughput/steps_per_second": 0.40878073659893077, "throughput/raw_tokens_per_second": 428626.63258007227, "throughput/peak_reserved_gib": 24.353515625}
|
| 58 |
+
{"step": 20, "stop_at": 954, "target": 128747, "training/loss": 9.758108884096146, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.758108884096146, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.1056271556229097, "training/generator_lr": 4.666666666666667e-05, "training/grad_norm": 0.07833481580018997, "training/positions": 20971520, "training/nonpadding_tokens": 20971118, "throughput/steps_per_second": 0.4090968275727434, "throughput/raw_tokens_per_second": 428962.6104293586, "throughput/peak_reserved_gib": 24.353515625}
|
| 59 |
+
{"step": 30, "stop_at": 954, "target": 128747, "training/loss": 9.68149453625083, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.68149453625083, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 0.21980331573996656, "training/generator_lr": 6.88888888888889e-05, "training/grad_norm": 0.07907956093549728, "training/positions": 31457280, "training/nonpadding_tokens": 31456759, "throughput/steps_per_second": 0.4082934124142342, "throughput/raw_tokens_per_second": 428121.81452406035, "throughput/peak_reserved_gib": 24.353515625}
|
| 60 |
+
{"step": 40, "stop_at": 954, "target": 128747, "training/loss": 9.57976196333766, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.57976196333766, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 1.08195939590301, "training/generator_lr": 9.111111111111112e-05, "training/grad_norm": 0.11145439743995667, "training/positions": 41943040, "training/nonpadding_tokens": 41942305, "throughput/steps_per_second": 0.4078265340731534, "throughput/raw_tokens_per_second": 427628.3883044617, "throughput/peak_reserved_gib": 24.353515625}
|
| 61 |
+
{"step": 50, "stop_at": 954, "target": 128747, "training/loss": 9.461714254319668, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.461714254319668, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.035468162102843, "training/generator_lr": 0.00011333333333333333, "training/grad_norm": 0.1421211063861847, "training/positions": 52428800, "training/nonpadding_tokens": 52427940, "throughput/steps_per_second": 0.4076745500473571, "throughput/raw_tokens_per_second": 427472.6530585819, "throughput/peak_reserved_gib": 24.353515625}
|
| 62 |
+
{"step": 60, "stop_at": 954, "target": 128747, "training/loss": 9.337426345050336, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.337426345050336, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 2.8979067595108696, "training/generator_lr": 0.00013555555555555556, "training/grad_norm": 0.20788517594337463, "training/positions": 62914560, "training/nonpadding_tokens": 62913625, "throughput/steps_per_second": 0.40751316291721795, "throughput/raw_tokens_per_second": 427305.4659703629, "throughput/peak_reserved_gib": 24.353515625}
|
| 63 |
+
{"step": 70, "stop_at": 954, "target": 128747, "training/loss": 9.213042849302292, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.213042849302292, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 3.865915355873746, "training/generator_lr": 0.00015777777777777776, "training/grad_norm": 0.21525025367736816, "training/positions": 73400320, "training/nonpadding_tokens": 73399129, "throughput/steps_per_second": 0.4075941311230782, "throughput/raw_tokens_per_second": 427382.9892267561, "throughput/peak_reserved_gib": 24.353515625}
|
| 64 |
+
{"step": 80, "stop_at": 954, "target": 128747, "training/loss": 9.090578303486108, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 9.090578303486108, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 4.84907700146321, "training/generator_lr": 0.00017999999999999998, "training/grad_norm": 0.3867489695549011, "training/positions": 83886080, "training/nonpadding_tokens": 83884677, "throughput/steps_per_second": 0.4075134760037024, "throughput/raw_tokens_per_second": 427300.211328367, "throughput/peak_reserved_gib": 24.353515625}
|
| 65 |
+
{"step": 90, "stop_at": 954, "target": 128747, "training/loss": 8.976701902598142, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.976701902598142, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 5.733659725125418, "training/generator_lr": 0.00020222222222222223, "training/grad_norm": 0.3807816505432129, "training/positions": 94371840, "training/nonpadding_tokens": 94370271, "throughput/steps_per_second": 0.40732012551424834, "throughput/raw_tokens_per_second": 427099.34641714493, "throughput/peak_reserved_gib": 24.353515625}
|
| 66 |
+
{"step": 100, "stop_at": 954, "target": 128747, "training/loss": 8.86207006573677, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.86207006573677, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 6.5993317568979935, "training/generator_lr": 0.00022444444444444446, "training/grad_norm": 0.4977039098739624, "training/positions": 104857600, "training/nonpadding_tokens": 104855825, "throughput/steps_per_second": 0.4071948106693431, "throughput/raw_tokens_per_second": 426966.3175793173, "throughput/peak_reserved_gib": 24.353515625}
|
| 67 |
+
{"step": 110, "stop_at": 954, "target": 128747, "training/loss": 8.749580790102481, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.749580790102481, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 7.371951100020903, "training/generator_lr": 0.0002466666666666667, "training/grad_norm": 0.4239509403705597, "training/positions": 115343360, "training/nonpadding_tokens": 115341579, "throughput/steps_per_second": 0.4072182521582959, "throughput/raw_tokens_per_second": 426999.04164418596, "throughput/peak_reserved_gib": 24.353515625}
|
| 68 |
+
{"step": 120, "stop_at": 954, "target": 128747, "training/loss": 8.632131148129702, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.632131148129702, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 8.165880082044314, "training/generator_lr": 0.00026888888888888893, "training/grad_norm": 0.5683702230453491, "training/positions": 125829120, "training/nonpadding_tokens": 125827327, "throughput/steps_per_second": 0.40711013239356286, "throughput/raw_tokens_per_second": 426885.4256525537, "throughput/peak_reserved_gib": 24.353515625}
|
| 69 |
+
{"step": 130, "stop_at": 954, "target": 128747, "training/loss": 8.525955478847028, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.525955478847028, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 8.761098191889632, "training/generator_lr": 0.00029111111111111113, "training/grad_norm": 0.4249029755592346, "training/positions": 136314880, "training/nonpadding_tokens": 136312889, "throughput/steps_per_second": 0.40740269695090114, "throughput/raw_tokens_per_second": 427184.6237845885, "throughput/peak_reserved_gib": 24.353515625}
|
| 70 |
+
{"step": 140, "stop_at": 954, "target": 128747, "training/loss": 8.407276231050492, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.407276231050492, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 9.426179713628763, "training/generator_lr": 0.0003133333333333334, "training/grad_norm": 0.3991111218929291, "training/positions": 146800640, "training/nonpadding_tokens": 146798407, "throughput/steps_per_second": 0.40755082063928466, "throughput/raw_tokens_per_second": 427338.1465727991, "throughput/peak_reserved_gib": 24.353515625}
|
| 71 |
+
{"step": 150, "stop_at": 954, "target": 128747, "training/loss": 8.29543913975358, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.29543913975358, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 10.021079379180602, "training/generator_lr": 0.0003355555555555556, "training/grad_norm": 0.5228633284568787, "training/positions": 157286400, "training/nonpadding_tokens": 157284143, "throughput/steps_per_second": 0.40769261859041633, "throughput/raw_tokens_per_second": 427495.7167687798, "throughput/peak_reserved_gib": 24.353515625}
|
| 72 |
+
{"step": 160, "stop_at": 954, "target": 128747, "training/loss": 8.203239648789168, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.203239648789168, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 10.39951204535953, "training/generator_lr": 0.00035777777777777777, "training/grad_norm": 0.44612228870391846, "training/positions": 167772160, "training/nonpadding_tokens": 167769767, "throughput/steps_per_second": 0.4075150188238271, "throughput/raw_tokens_per_second": 427304.9261739573, "throughput/peak_reserved_gib": 24.353515625}
|
| 73 |
+
{"step": 170, "stop_at": 954, "target": 128747, "training/loss": 8.088917724043132, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 8.088917724043132, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 10.923204300794314, "training/generator_lr": 0.00038, "training/grad_norm": 0.3123162090778351, "training/positions": 178257920, "training/nonpadding_tokens": 178255314, "throughput/steps_per_second": 0.4074000595940846, "throughput/raw_tokens_per_second": 427181.2472676575, "throughput/peak_reserved_gib": 24.353515625}
|
| 74 |
+
{"step": 180, "stop_at": 954, "target": 128747, "training/loss": 7.975613238289952, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.975613238289952, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 11.483646660744148, "training/generator_lr": 0.0004022222222222222, "training/grad_norm": 0.3921535313129425, "training/positions": 188743680, "training/nonpadding_tokens": 188740985, "throughput/steps_per_second": 0.40752568729295824, "throughput/raw_tokens_per_second": 427318.02810028405, "throughput/peak_reserved_gib": 24.353515625}
|
| 75 |
+
{"step": 190, "stop_at": 954, "target": 128747, "training/loss": 7.8878518410027025, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.8878518410027025, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 11.85516663618311, "training/generator_lr": 0.00042444444444444447, "training/grad_norm": 0.4721970856189728, "training/positions": 199229440, "training/nonpadding_tokens": 199226665, "throughput/steps_per_second": 0.4075858935362929, "throughput/raw_tokens_per_second": 427381.5252135635, "throughput/peak_reserved_gib": 24.353515625}
|
| 76 |
+
{"step": 200, "stop_at": 954, "target": 128747, "training/loss": 7.786064530536533, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.786064530536533, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 12.304418047136288, "training/generator_lr": 0.00044666666666666666, "training/grad_norm": 0.39285171031951904, "training/positions": 209715200, "training/nonpadding_tokens": 209712037, "throughput/steps_per_second": 0.4075740987969532, "throughput/raw_tokens_per_second": 427356.6043450807, "throughput/peak_reserved_gib": 24.353515625}
|
| 77 |
+
{"step": 210, "stop_at": 954, "target": 128747, "training/loss": 7.696019466221332, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.696019466221332, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 12.69830097721572, "training/generator_lr": 0.0004688888888888889, "training/grad_norm": 0.40060362219810486, "training/positions": 220200960, "training/nonpadding_tokens": 220197684, "throughput/steps_per_second": 0.4077566952825902, "throughput/raw_tokens_per_second": 427559.2768619806, "throughput/peak_reserved_gib": 24.353515625}
|
| 78 |
+
{"step": 220, "stop_at": 954, "target": 128747, "training/loss": 7.613952485099435, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.613952485099435, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 13.061198461015886, "training/generator_lr": 0.0004911111111111111, "training/grad_norm": 0.48230910301208496, "training/positions": 230686720, "training/nonpadding_tokens": 230683308, "throughput/steps_per_second": 0.40797608596128954, "throughput/raw_tokens_per_second": 427788.3838381761, "throughput/peak_reserved_gib": 24.353515625}
|
| 79 |
+
{"step": 230, "stop_at": 954, "target": 128747, "training/loss": 7.529779851436615, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.529779851436615, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 13.45795065583194, "training/generator_lr": 0.0004999999981701273, "training/grad_norm": 0.4334399402141571, "training/positions": 241172480, "training/nonpadding_tokens": 241168894, "throughput/steps_per_second": 0.40782563332240546, "throughput/raw_tokens_per_second": 427629.07512065483, "throughput/peak_reserved_gib": 24.353515625}
|
| 80 |
+
{"step": 240, "stop_at": 954, "target": 128747, "training/loss": 7.45125268958509, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.45125268958509, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 13.84993925062709, "training/generator_lr": 0.0004999999835311453, "training/grad_norm": 0.28200238943099976, "training/positions": 251658240, "training/nonpadding_tokens": 251654582, "throughput/steps_per_second": 0.4077266794327132, "throughput/raw_tokens_per_second": 427529.47498074477, "throughput/peak_reserved_gib": 24.353515625}
|
| 81 |
+
{"step": 250, "stop_at": 954, "target": 128747, "training/loss": 7.365950198844075, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.365950198844075, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 14.326307417955686, "training/generator_lr": 0.0004999999542531824, "training/grad_norm": 0.3151077926158905, "training/positions": 262144000, "training/nonpadding_tokens": 262140041, "throughput/steps_per_second": 0.40782853845189965, "throughput/raw_tokens_per_second": 427626.94189673173, "throughput/peak_reserved_gib": 24.353515625}
|
| 82 |
+
{"step": 260, "stop_at": 954, "target": 128747, "training/loss": 7.301213253289461, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.301213253289461, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 14.682623719690636, "training/generator_lr": 0.0004999999103362402, "training/grad_norm": 0.3669530749320984, "training/positions": 272629760, "training/nonpadding_tokens": 272625566, "throughput/steps_per_second": 0.4079094089290304, "throughput/raw_tokens_per_second": 427714.43050605716, "throughput/peak_reserved_gib": 24.353515625}
|
| 83 |
+
{"step": 270, "stop_at": 954, "target": 128747, "training/loss": 7.243592227250337, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.243592227250337, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 14.989840810514215, "training/generator_lr": 0.0004999998517803213, "training/grad_norm": 0.3860306143760681, "training/positions": 283115520, "training/nonpadding_tokens": 283111224, "throughput/steps_per_second": 0.4079417160895974, "throughput/raw_tokens_per_second": 427753.73188486154, "throughput/peak_reserved_gib": 24.353515625}
|
| 84 |
+
{"step": 280, "stop_at": 954, "target": 128747, "training/loss": 7.188855135068297, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.188855135068297, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 15.290849054138796, "training/generator_lr": 0.0004999997785854293, "training/grad_norm": 0.32739344239234924, "training/positions": 293601280, "training/nonpadding_tokens": 293596820, "throughput/steps_per_second": 0.40803122481956255, "throughput/raw_tokens_per_second": 427845.0578843106, "throughput/peak_reserved_gib": 24.353515625}
|
| 85 |
+
{"step": 290, "stop_at": 954, "target": 128747, "training/loss": 7.131561341509223, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.131561341509223, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 15.657904277278428, "training/generator_lr": 0.0004999996907515685, "training/grad_norm": 0.3538442552089691, "training/positions": 304087040, "training/nonpadding_tokens": 304082455, "throughput/steps_per_second": 0.407851060296959, "throughput/raw_tokens_per_second": 427657.7352636904, "throughput/peak_reserved_gib": 24.353515625}
|
| 86 |
+
{"step": 300, "stop_at": 954, "target": 128747, "training/loss": 7.08795943967998, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.08795943967998, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 15.902600138482441, "training/generator_lr": 0.0004999995882787442, "training/grad_norm": 0.3638913035392761, "training/positions": 314572800, "training/nonpadding_tokens": 314568065, "throughput/steps_per_second": 0.4079842800878165, "throughput/raw_tokens_per_second": 427796.40471316094, "throughput/peak_reserved_gib": 24.353515625}
|
| 87 |
+
{"step": 310, "stop_at": 954, "target": 128747, "training/loss": 7.039632910117507, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 7.039632910117507, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 16.212116560409697, "training/generator_lr": 0.0004999994711669624, "training/grad_norm": 0.2915151119232178, "training/positions": 325058560, "training/nonpadding_tokens": 325053596, "throughput/steps_per_second": 0.4081033122643531, "throughput/raw_tokens_per_second": 427917.99319505546, "throughput/peak_reserved_gib": 24.353515625}
|
| 88 |
+
{"step": 320, "stop_at": 954, "target": 128747, "training/loss": 6.990682903677225, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.990682903677225, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 16.546791714569398, "training/generator_lr": 0.0004999993394162303, "training/grad_norm": 0.33340489864349365, "training/positions": 335544320, "training/nonpadding_tokens": 335538862, "throughput/steps_per_second": 0.4079292526609766, "throughput/raw_tokens_per_second": 427724.67233315477, "throughput/peak_reserved_gib": 24.353515625}
|
| 89 |
+
{"step": 330, "stop_at": 954, "target": 128747, "training/loss": 6.954669648781419, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.954669648781419, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 16.750035927048494, "training/generator_lr": 0.0004999991930265556, "training/grad_norm": 0.3257168233394623, "training/positions": 346030080, "training/nonpadding_tokens": 346024551, "throughput/steps_per_second": 0.40781841698230986, "throughput/raw_tokens_per_second": 427625.70889488194, "throughput/peak_reserved_gib": 24.353515625}
|
| 90 |
+
{"step": 340, "stop_at": 954, "target": 128747, "training/loss": 6.912285809591412, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.912285809591412, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 17.04010765050167, "training/generator_lr": 0.000499999031997947, "training/grad_norm": 0.29693424701690674, "training/positions": 356515840, "training/nonpadding_tokens": 356510227, "throughput/steps_per_second": 0.4079570569343417, "throughput/raw_tokens_per_second": 427770.5520927061, "throughput/peak_reserved_gib": 24.353515625}
|
| 91 |
+
{"step": 350, "stop_at": 954, "target": 128747, "training/loss": 6.87697426006198, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.87697426006198, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 17.26875228626672, "training/generator_lr": 0.0004999988563304143, "training/grad_norm": 0.28626590967178345, "training/positions": 367001600, "training/nonpadding_tokens": 366995799, "throughput/steps_per_second": 0.40798061340119723, "throughput/raw_tokens_per_second": 427791.0096422419, "throughput/peak_reserved_gib": 24.353515625}
|
| 92 |
+
{"step": 360, "stop_at": 954, "target": 128747, "training/loss": 6.833841783553362, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.833841783553362, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 17.58629840353261, "training/generator_lr": 0.000499998666023968, "training/grad_norm": 0.3001255989074707, "training/positions": 377487360, "training/nonpadding_tokens": 377481238, "throughput/steps_per_second": 0.4078829764988133, "throughput/raw_tokens_per_second": 427683.206921674, "throughput/peak_reserved_gib": 24.353515625}
|
| 93 |
+
{"step": 370, "stop_at": 954, "target": 128747, "training/loss": 6.805006217211485, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.805006217211485, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 17.766982062604516, "training/generator_lr": 0.0004999984610786194, "training/grad_norm": 0.4011838436126709, "training/positions": 387973120, "training/nonpadding_tokens": 387966997, "throughput/steps_per_second": 0.4078750073193625, "throughput/raw_tokens_per_second": 427687.9028874071, "throughput/peak_reserved_gib": 24.353515625}
|
| 94 |
+
{"step": 380, "stop_at": 954, "target": 128747, "training/loss": 6.774790311604738, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.774790311604738, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 17.9812705764005, "training/generator_lr": 0.0004999982414943807, "training/grad_norm": 0.26410797238349915, "training/positions": 398458880, "training/nonpadding_tokens": 398452694, "throughput/steps_per_second": 0.40793377223892036, "throughput/raw_tokens_per_second": 427746.993176433, "throughput/peak_reserved_gib": 24.353515625}
|
| 95 |
+
{"step": 390, "stop_at": 954, "target": 128747, "training/loss": 6.7378301825374365, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.7378301825374365, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 18.27435690583194, "training/generator_lr": 0.000499998007271265, "training/grad_norm": 0.33278515934944153, "training/positions": 408944640, "training/nonpadding_tokens": 408938400, "throughput/steps_per_second": 0.40779223984774227, "throughput/raw_tokens_per_second": 427598.953612491, "throughput/peak_reserved_gib": 24.353515625}
|
| 96 |
+
{"step": 400, "stop_at": 954, "target": 128747, "training/loss": 6.709629044309258, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.709629044309258, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 18.463891683214882, "training/generator_lr": 0.0004999977584092864, "training/grad_norm": 0.3659369647502899, "training/positions": 419430400, "training/nonpadding_tokens": 419424058, "throughput/steps_per_second": 0.4079471561660412, "throughput/raw_tokens_per_second": 427759.4361629699, "throughput/peak_reserved_gib": 24.353515625}
|
| 97 |
+
{"step": 410, "stop_at": 954, "target": 128747, "training/loss": 6.682891883701086, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.682891883701086, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 18.68603352320234, "training/generator_lr": 0.0004999974949084598, "training/grad_norm": 0.2996213138103485, "training/positions": 429916160, "training/nonpadding_tokens": 429909611, "throughput/steps_per_second": 0.4079384351159854, "throughput/raw_tokens_per_second": 427746.00821457256, "throughput/peak_reserved_gib": 24.353515625}
|
| 98 |
+
{"step": 420, "stop_at": 954, "target": 128747, "training/loss": 6.656715175881982, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.656715175881982, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 18.903906576609533, "training/generator_lr": 0.0004999972167688007, "training/grad_norm": 0.27686527371406555, "training/positions": 440401920, "training/nonpadding_tokens": 440395082, "throughput/steps_per_second": 0.40777130184960664, "throughput/raw_tokens_per_second": 427567.4160176297, "throughput/peak_reserved_gib": 24.353515625}
|
| 99 |
+
{"step": 430, "stop_at": 954, "target": 128747, "training/loss": 6.628474683687091, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.628474683687091, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 19.12791499007107, "training/generator_lr": 0.000499996923990326, "training/grad_norm": 0.34010064601898193, "training/positions": 450887680, "training/nonpadding_tokens": 450880746, "throughput/steps_per_second": 0.40780801073144635, "throughput/raw_tokens_per_second": 427613.77770383406, "throughput/peak_reserved_gib": 24.353515625}
|
| 100 |
+
{"step": 440, "stop_at": 954, "target": 128747, "training/loss": 6.601665687188506, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.601665687188506, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 19.331796091137125, "training/generator_lr": 0.0004999966165730531, "training/grad_norm": 0.30975162982940674, "training/positions": 461373440, "training/nonpadding_tokens": 461366347, "throughput/steps_per_second": 0.40784562045456396, "throughput/raw_tokens_per_second": 427650.64456839964, "throughput/peak_reserved_gib": 24.353515625}
|
| 101 |
+
{"step": 450, "stop_at": 954, "target": 128747, "training/loss": 6.576487730816007, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.576487730816007, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 19.565821618938127, "training/generator_lr": 0.0004999962945170003, "training/grad_norm": 0.27116572856903076, "training/positions": 471859200, "training/nonpadding_tokens": 471852012, "throughput/steps_per_second": 0.40788407749526484, "throughput/raw_tokens_per_second": 427693.57954493864, "throughput/peak_reserved_gib": 24.353515625}
|
| 102 |
+
{"step": 460, "stop_at": 954, "target": 128747, "training/loss": 6.554621310532093, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.554621310532093, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 19.76224949048913, "training/generator_lr": 0.0004999959578221868, "training/grad_norm": 0.29644080996513367, "training/positions": 482344960, "training/nonpadding_tokens": 482337515, "throughput/steps_per_second": 0.40806854506828844, "throughput/raw_tokens_per_second": 427880.39535191737, "throughput/peak_reserved_gib": 24.353515625}
|
| 103 |
+
{"step": 470, "stop_at": 954, "target": 128747, "training/loss": 6.538460053503513, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.538460053503513, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 19.908497073578594, "training/generator_lr": 0.000499995606488633, "training/grad_norm": 0.3355327844619751, "training/positions": 492830720, "training/nonpadding_tokens": 492823060, "throughput/steps_per_second": 0.40815376516908275, "throughput/raw_tokens_per_second": 427971.467159985, "throughput/peak_reserved_gib": 24.353515625}
|
| 104 |
+
{"step": 480, "stop_at": 954, "target": 128747, "training/loss": 6.5155995462089775, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.5155995462089775, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 20.145022797345316, "training/generator_lr": 0.0004999952405163596, "training/grad_norm": 0.30374762415885925, "training/positions": 503316480, "training/nonpadding_tokens": 503308713, "throughput/steps_per_second": 0.4082228818760131, "throughput/raw_tokens_per_second": 428048.34860118624, "throughput/peak_reserved_gib": 24.353515625}
|
| 105 |
+
{"step": 490, "stop_at": 954, "target": 128747, "training/loss": 6.494265393912793, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.494265393912793, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 20.313483094690636, "training/generator_lr": 0.0004999948599053887, "training/grad_norm": 0.29466313123703003, "training/positions": 513802240, "training/nonpadding_tokens": 513794206, "throughput/steps_per_second": 0.40807144638264453, "throughput/raw_tokens_per_second": 427883.0294545095, "throughput/peak_reserved_gib": 24.353515625}
|
| 106 |
+
{"step": 500, "stop_at": 954, "target": 128747, "training/loss": 6.473226903006434, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.473226903006434, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 20.51648888221154, "training/generator_lr": 0.0004999944646557427, "training/grad_norm": 0.3445967733860016, "training/positions": 524288000, "training/nonpadding_tokens": 524279876, "throughput/steps_per_second": 0.4081675584460357, "throughput/raw_tokens_per_second": 427991.03225708427, "throughput/peak_reserved_gib": 24.353515625}
|
| 107 |
+
{"step": 510, "stop_at": 954, "target": 128747, "training/loss": 6.45149132758379, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.45149132758379, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 20.738042825041806, "training/generator_lr": 0.0004999940547674456, "training/grad_norm": 0.32744285464286804, "training/positions": 534773760, "training/nonpadding_tokens": 534765323, "throughput/steps_per_second": 0.4082362205511829, "throughput/raw_tokens_per_second": 428053.9254069739, "throughput/peak_reserved_gib": 24.353515625}
|
| 108 |
+
{"step": 520, "stop_at": 954, "target": 128747, "training/loss": 6.442303481325507, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.442303481325507, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 20.86348048181438, "training/generator_lr": 0.0004999936302405217, "training/grad_norm": 0.36089566349983215, "training/positions": 545259520, "training/nonpadding_tokens": 545250857, "throughput/steps_per_second": 0.4082438585924584, "throughput/raw_tokens_per_second": 428065.48595624143, "throughput/peak_reserved_gib": 24.353515625}
|
| 109 |
+
{"step": 530, "stop_at": 954, "target": 128747, "training/loss": 6.4238133382052185, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.4238133382052185, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 21.010505395589465, "training/generator_lr": 0.0004999931910749963, "training/grad_norm": 0.3506404459476471, "training/positions": 555745280, "training/nonpadding_tokens": 555736516, "throughput/steps_per_second": 0.40847211350533674, "throughput/raw_tokens_per_second": 428309.92932262557, "throughput/peak_reserved_gib": 24.353515625}
|
| 110 |
+
{"step": 540, "stop_at": 954, "target": 128747, "training/loss": 6.404988915845752, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.404988915845752, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 21.23909940687709, "training/generator_lr": 0.0004999927372708958, "training/grad_norm": 0.3335840404033661, "training/positions": 566231040, "training/nonpadding_tokens": 566221693, "throughput/steps_per_second": 0.4082049319666474, "throughput/raw_tokens_per_second": 428010.0963943256, "throughput/peak_reserved_gib": 24.353515625}
|
| 111 |
+
{"step": 550, "stop_at": 954, "target": 128747, "training/loss": 6.386105476692319, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.386105476692319, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 21.446120858591136, "training/generator_lr": 0.0004999922688282472, "training/grad_norm": 0.31673121452331543, "training/positions": 576716800, "training/nonpadding_tokens": 576707372, "throughput/steps_per_second": 0.4081613789841912, "throughput/raw_tokens_per_second": 427984.9200225575, "throughput/peak_reserved_gib": 24.353515625}
|
| 112 |
+
{"step": 560, "stop_at": 954, "target": 128747, "training/loss": 6.37326210141182, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.37326210141182, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 21.59482781145485, "training/generator_lr": 0.0004999917857470785, "training/grad_norm": 0.47669023275375366, "training/positions": 587202560, "training/nonpadding_tokens": 587192883, "throughput/steps_per_second": 0.4080445330410536, "throughput/raw_tokens_per_second": 427855.5439691831, "throughput/peak_reserved_gib": 24.353515625}
|
| 113 |
+
{"step": 570, "stop_at": 954, "target": 128747, "training/loss": 6.361703063920141, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.361703063920141, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 21.665608996132942, "training/generator_lr": 0.0004999912880274186, "training/grad_norm": 0.2572002112865448, "training/positions": 597688320, "training/nonpadding_tokens": 597678457, "throughput/steps_per_second": 0.4082042396981426, "throughput/raw_tokens_per_second": 428025.57624686125, "throughput/peak_reserved_gib": 24.353515625}
|
| 114 |
+
{"step": 580, "stop_at": 954, "target": 128747, "training/loss": 6.336093625053763, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.336093625053763, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 22.01889926055602, "training/generator_lr": 0.0004999907756692973, "training/grad_norm": 0.3296107053756714, "training/positions": 608174080, "training/nonpadding_tokens": 608164023, "throughput/steps_per_second": 0.4082555801580668, "throughput/raw_tokens_per_second": 428079.08306157, "throughput/peak_reserved_gib": 24.353515625}
|
| 115 |
+
{"step": 590, "stop_at": 954, "target": 128747, "training/loss": 6.332702476903796, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.332702476903796, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 22.069010961015888, "training/generator_lr": 0.0004999902486727452, "training/grad_norm": 0.3540639579296112, "training/positions": 618659840, "training/nonpadding_tokens": 618649723, "throughput/steps_per_second": 0.40814474994463523, "throughput/raw_tokens_per_second": 427968.3404494462, "throughput/peak_reserved_gib": 24.353515625}
|
| 116 |
+
{"step": 600, "stop_at": 954, "target": 128747, "training/loss": 6.31470256857574, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.31470256857574, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 22.28083357284699, "training/generator_lr": 0.0004999897070377936, "training/grad_norm": 0.29616010189056396, "training/positions": 629145600, "training/nonpadding_tokens": 629135483, "throughput/steps_per_second": 0.408169777080186, "throughput/raw_tokens_per_second": 427997.0321716331, "throughput/peak_reserved_gib": 24.353515625}
|
| 117 |
+
{"step": 610, "stop_at": 954, "target": 128747, "training/loss": 6.299891290068627, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.299891290068627, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 22.466300428511705, "training/generator_lr": 0.0004999891507644751, "training/grad_norm": 0.3090231120586395, "training/positions": 639631360, "training/nonpadding_tokens": 639621155, "throughput/steps_per_second": 0.40810495485361087, "throughput/raw_tokens_per_second": 427925.46981697716, "throughput/peak_reserved_gib": 24.353515625}
|
| 118 |
+
{"step": 620, "stop_at": 954, "target": 128747, "training/loss": 6.284555285423994, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.284555285423994, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 22.616434665029264, "training/generator_lr": 0.0004999885798528228, "training/grad_norm": 0.2973569929599762, "training/positions": 650117120, "training/nonpadding_tokens": 650106618, "throughput/steps_per_second": 0.40796797469997015, "throughput/raw_tokens_per_second": 427773.3103901473, "throughput/peak_reserved_gib": 24.353515625}
|
| 119 |
+
{"step": 630, "stop_at": 954, "target": 128747, "training/loss": 6.271211093664169, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.271211093664169, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 22.807497975020905, "training/generator_lr": 0.0004999879943028709, "training/grad_norm": 0.3545754551887512, "training/positions": 660602880, "training/nonpadding_tokens": 660592301, "throughput/steps_per_second": 0.4081388023659573, "throughput/raw_tokens_per_second": 427961.41016090784, "throughput/peak_reserved_gib": 24.353515625}
|
| 120 |
+
{"step": 640, "stop_at": 954, "target": 128747, "training/loss": 6.261996277421713, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.261996277421713, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 22.91022810409699, "training/generator_lr": 0.0004999873941146543, "training/grad_norm": 0.29381394386291504, "training/positions": 671088640, "training/nonpadding_tokens": 671078026, "throughput/steps_per_second": 0.4080801287405791, "throughput/raw_tokens_per_second": 427901.6007938309, "throughput/peak_reserved_gib": 24.353515625}
|
| 121 |
+
{"step": 650, "stop_at": 954, "target": 128747, "training/loss": 6.244654889032245, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.244654889032245, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 23.120589138273413, "training/generator_lr": 0.0004999867792882088, "training/grad_norm": 0.3419444262981415, "training/positions": 681574400, "training/nonpadding_tokens": 681563648, "throughput/steps_per_second": 0.40814758945658974, "throughput/raw_tokens_per_second": 427968.13432529854, "throughput/peak_reserved_gib": 24.353515625}
|
| 122 |
+
{"step": 660, "stop_at": 954, "target": 128747, "training/loss": 6.237538556754589, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.237538556754589, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 23.26037965091973, "training/generator_lr": 0.0004999861498235714, "training/grad_norm": 0.3288504183292389, "training/positions": 692060160, "training/nonpadding_tokens": 692049393, "throughput/steps_per_second": 0.40827164809719924, "throughput/raw_tokens_per_second": 428103.2392676966, "throughput/peak_reserved_gib": 24.353515625}
|
| 123 |
+
{"step": 670, "stop_at": 954, "target": 128747, "training/loss": 6.21738072745502, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.21738072745502, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 23.459569006061873, "training/generator_lr": 0.0004999855057207795, "training/grad_norm": 0.3211475610733032, "training/positions": 702545920, "training/nonpadding_tokens": 702535094, "throughput/steps_per_second": 0.4081708481875891, "throughput/raw_tokens_per_second": 427995.7471011451, "throughput/peak_reserved_gib": 24.353515625}
|
| 124 |
+
{"step": 680, "stop_at": 954, "target": 128747, "training/loss": 6.212292714044452, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.212292714044452, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 23.56590490436873, "training/generator_lr": 0.0004999848469798716, "training/grad_norm": 0.292034775018692, "training/positions": 713031680, "training/nonpadding_tokens": 713020784, "throughput/steps_per_second": 0.4081315178547683, "throughput/raw_tokens_per_second": 427954.05754545657, "throughput/peak_reserved_gib": 24.353515625}
|
| 125 |
+
{"step": 690, "stop_at": 954, "target": 128747, "training/loss": 6.196095773950219, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.196095773950219, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 23.792754820756688, "training/generator_lr": 0.0004999841736008871, "training/grad_norm": 0.2980763912200928, "training/positions": 723517440, "training/nonpadding_tokens": 723506441, "throughput/steps_per_second": 0.4082628436964252, "throughput/raw_tokens_per_second": 428090.41448453267, "throughput/peak_reserved_gib": 24.353515625}
|
| 126 |
+
{"step": 700, "stop_at": 954, "target": 128747, "training/loss": 6.187444163113833, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.187444163113833, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 23.91399227895067, "training/generator_lr": 0.0004999834855838663, "training/grad_norm": 0.35733169317245483, "training/positions": 734003200, "training/nonpadding_tokens": 733992123, "throughput/steps_per_second": 0.4084211290373464, "throughput/raw_tokens_per_second": 428257.40811665804, "throughput/peak_reserved_gib": 24.353515625}
|
| 127 |
+
{"step": 710, "stop_at": 954, "target": 128747, "training/loss": 6.1746968444436785, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.1746968444436785, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 24.091057104410535, "training/generator_lr": 0.0004999827829288502, "training/grad_norm": 0.3423677980899811, "training/positions": 744488960, "training/nonpadding_tokens": 744477778, "throughput/steps_per_second": 0.4082141486166307, "throughput/raw_tokens_per_second": 428039.2728512717, "throughput/peak_reserved_gib": 24.353515625}
|
| 128 |
+
{"step": 720, "stop_at": 954, "target": 128747, "training/loss": 6.159521489217878, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.159521489217878, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 24.268946618938127, "training/generator_lr": 0.0004999820656358808, "training/grad_norm": 0.329266220331192, "training/positions": 754974720, "training/nonpadding_tokens": 754963460, "throughput/steps_per_second": 0.40825966987806506, "throughput/raw_tokens_per_second": 428088.10717663687, "throughput/peak_reserved_gib": 24.353515625}
|
| 129 |
+
{"step": 730, "stop_at": 954, "target": 128747, "training/loss": 6.154356696084141, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.154356696084141, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 24.39796228313127, "training/generator_lr": 0.000499981333705001, "training/grad_norm": 0.3683840036392212, "training/positions": 765460480, "training/nonpadding_tokens": 765449028, "throughput/steps_per_second": 0.40810501795926096, "throughput/raw_tokens_per_second": 427921.2916953052, "throughput/peak_reserved_gib": 24.353515625}
|
| 130 |
+
{"step": 740, "stop_at": 954, "target": 128747, "training/loss": 6.1419789209961895, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.1419789209961895, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 24.556640625, "training/generator_lr": 0.0004999805871362545, "training/grad_norm": 0.3246363401412964, "training/positions": 775946240, "training/nonpadding_tokens": 775934485, "throughput/steps_per_second": 0.4076785591448207, "throughput/raw_tokens_per_second": 427469.60017349746, "throughput/peak_reserved_gib": 24.353515625}
|
| 131 |
+
{"step": 750, "stop_at": 954, "target": 128747, "training/loss": 6.128755298629403, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.128755298629403, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 24.69614535430602, "training/generator_lr": 0.000499979825929686, "training/grad_norm": 0.3538039028644562, "training/positions": 786432000, "training/nonpadding_tokens": 786420144, "throughput/steps_per_second": 0.4079538594428746, "throughput/raw_tokens_per_second": 427766.5057851913, "throughput/peak_reserved_gib": 24.353515625}
|
| 132 |
+
{"step": 760, "stop_at": 954, "target": 128747, "training/loss": 6.122572975605726, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.122572975605726, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 24.829754781563544, "training/generator_lr": 0.0004999790500853408, "training/grad_norm": 0.3228318989276886, "training/positions": 796917760, "training/nonpadding_tokens": 796905882, "throughput/steps_per_second": 0.40842656325822013, "throughput/raw_tokens_per_second": 428265.3934566123, "throughput/peak_reserved_gib": 24.353515625}
|
| 133 |
+
{"step": 770, "stop_at": 954, "target": 128747, "training/loss": 6.114674207195639, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.114674207195639, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 24.984018995610366, "training/generator_lr": 0.0004999782596032654, "training/grad_norm": 0.3554404377937317, "training/positions": 807403520, "training/nonpadding_tokens": 807391508, "throughput/steps_per_second": 0.40848777206060505, "throughput/raw_tokens_per_second": 428325.0003400754, "throughput/peak_reserved_gib": 24.353515625}
|
| 134 |
+
{"step": 780, "stop_at": 954, "target": 128747, "training/loss": 6.104674210771918, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.104674210771918, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 25.084395903010034, "training/generator_lr": 0.000499977454483507, "training/grad_norm": 0.3030296862125397, "training/positions": 817889280, "training/nonpadding_tokens": 817877191, "throughput/steps_per_second": 0.40841470345111025, "throughput/raw_tokens_per_second": 428250.7112927348, "throughput/peak_reserved_gib": 24.353515625}
|
| 135 |
+
{"step": 790, "stop_at": 954, "target": 128747, "training/loss": 6.089097730815411, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.089097730815411, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 25.3109796326296, "training/generator_lr": 0.0004999766347261137, "training/grad_norm": 0.32302308082580566, "training/positions": 828375040, "training/nonpadding_tokens": 828362696, "throughput/steps_per_second": 0.40839378000809595, "throughput/raw_tokens_per_second": 428221.50222437904, "throughput/peak_reserved_gib": 24.353515625}
|
| 136 |
+
{"step": 800, "stop_at": 954, "target": 128747, "training/loss": 6.085591481253505, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.085591481253505, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 25.37402180445234, "training/generator_lr": 0.0004999758003311345, "training/grad_norm": 0.3881380259990692, "training/positions": 838860800, "training/nonpadding_tokens": 838848410, "throughput/steps_per_second": 0.40824288756606275, "throughput/raw_tokens_per_second": 428071.816155189, "throughput/peak_reserved_gib": 24.353515625}
|
| 137 |
+
{"step": 810, "stop_at": 954, "target": 128747, "training/loss": 6.079013765603304, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.079013765603304, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 25.476895641722408, "training/generator_lr": 0.0004999749512986192, "training/grad_norm": 0.3493039906024933, "training/positions": 849346560, "training/nonpadding_tokens": 849334036, "throughput/steps_per_second": 0.40803115690845315, "throughput/raw_tokens_per_second": 427846.2107689356, "throughput/peak_reserved_gib": 24.353515625}
|
| 138 |
+
{"step": 820, "stop_at": 954, "target": 128747, "training/loss": 6.066323831304908, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.066323831304908, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 25.653424827550168, "training/generator_lr": 0.0004999740876286187, "training/grad_norm": 0.2783434987068176, "training/positions": 859832320, "training/nonpadding_tokens": 859819502, "throughput/steps_per_second": 0.4081650901040861, "throughput/raw_tokens_per_second": 427980.1174673331, "throughput/peak_reserved_gib": 24.353515625}
|
| 139 |
+
{"step": 830, "stop_at": 954, "target": 128747, "training/loss": 6.058590435236693, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.058590435236693, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 25.780302832357858, "training/generator_lr": 0.0004999732093211843, "training/grad_norm": 0.37396255135536194, "training/positions": 870318080, "training/nonpadding_tokens": 870305154, "throughput/steps_per_second": 0.40849724157981127, "throughput/raw_tokens_per_second": 428335.99181658315, "throughput/peak_reserved_gib": 24.353515625}
|
| 140 |
+
{"step": 840, "stop_at": 954, "target": 128747, "training/loss": 6.050515468046069, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.050515468046069, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 25.903929439276755, "training/generator_lr": 0.0004999723163763688, "training/grad_norm": 0.3242272734642029, "training/positions": 880803840, "training/nonpadding_tokens": 880790680, "throughput/steps_per_second": 0.40833531203998114, "throughput/raw_tokens_per_second": 428161.0531113335, "throughput/peak_reserved_gib": 24.353515625}
|
| 141 |
+
{"step": 850, "stop_at": 954, "target": 128747, "training/loss": 6.046826994046569, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.046826994046569, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 25.958487928511705, "training/generator_lr": 0.0004999714087942253, "training/grad_norm": 0.3176061809062958, "training/positions": 891289600, "training/nonpadding_tokens": 891276364, "throughput/steps_per_second": 0.408327636012695, "throughput/raw_tokens_per_second": 428159.455969614, "throughput/peak_reserved_gib": 24.353515625}
|
| 142 |
+
{"step": 860, "stop_at": 954, "target": 128747, "training/loss": 6.033342712372542, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.033342712372542, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 26.151662769126254, "training/generator_lr": 0.0004999704865748083, "training/grad_norm": 0.2960236072540283, "training/positions": 901775360, "training/nonpadding_tokens": 901761946, "throughput/steps_per_second": 0.4083186782781944, "throughput/raw_tokens_per_second": 428145.8983217626, "throughput/peak_reserved_gib": 24.353515625}
|
| 143 |
+
{"step": 870, "stop_at": 954, "target": 128747, "training/loss": 6.027253070473671, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.027253070473671, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 26.292504964464882, "training/generator_lr": 0.0004999695497181725, "training/grad_norm": 0.32039687037467957, "training/positions": 912261120, "training/nonpadding_tokens": 912247625, "throughput/steps_per_second": 0.4084255980845175, "throughput/raw_tokens_per_second": 428261.9716897265, "throughput/peak_reserved_gib": 24.353515625}
|
| 144 |
+
{"step": 880, "stop_at": 954, "target": 128747, "training/loss": 6.025838574394584, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.025838574394584, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 26.258235459343645, "training/generator_lr": 0.0004999685982243744, "training/grad_norm": 0.45321306586265564, "training/positions": 922746880, "training/nonpadding_tokens": 922733261, "throughput/steps_per_second": 0.4085342307788901, "throughput/raw_tokens_per_second": 428374.1237487438, "throughput/peak_reserved_gib": 24.353515625}
|
| 145 |
+
{"step": 890, "stop_at": 954, "target": 128747, "training/loss": 6.017437618598342, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.017437618598342, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 26.395744604410535, "training/generator_lr": 0.0004999676320934704, "training/grad_norm": 0.25396162271499634, "training/positions": 933232640, "training/nonpadding_tokens": 933218837, "throughput/steps_per_second": 0.40865885550793657, "throughput/raw_tokens_per_second": 428502.3487501487, "throughput/peak_reserved_gib": 24.353515625}
|
| 146 |
+
{"step": 900, "stop_at": 954, "target": 128747, "training/loss": 6.002003015205264, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 6.002003015205264, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 26.63646236151756, "training/generator_lr": 0.0004999666513255184, "training/grad_norm": 0.3259555995464325, "training/positions": 943718400, "training/nonpadding_tokens": 943704386, "throughput/steps_per_second": 0.4085716123207622, "throughput/raw_tokens_per_second": 428409.76609983563, "throughput/peak_reserved_gib": 24.353515625}
|
| 147 |
+
{"step": 910, "stop_at": 954, "target": 128747, "training/loss": 5.994752109050751, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 5.994752109050751, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 26.723881035744146, "training/generator_lr": 0.000499965655920577, "training/grad_norm": 0.3730374574661255, "training/positions": 954204160, "training/nonpadding_tokens": 954190088, "throughput/steps_per_second": 0.4085200836882867, "throughput/raw_tokens_per_second": 428361.98585704353, "throughput/peak_reserved_gib": 24.353515625}
|
| 148 |
+
{"step": 920, "stop_at": 954, "target": 128747, "training/loss": 5.990219716727734, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 5.990219716727734, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 26.795005160430602, "training/generator_lr": 0.0004999646458787058, "training/grad_norm": 0.35145512223243713, "training/positions": 964689920, "training/nonpadding_tokens": 964675810, "throughput/steps_per_second": 0.4082438274430648, "throughput/raw_tokens_per_second": 428073.12827839487, "throughput/peak_reserved_gib": 24.353515625}
|
| 149 |
+
{"step": 930, "stop_at": 954, "target": 128747, "training/loss": 5.98227760978043, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 5.98227760978043, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 26.969644910117058, "training/generator_lr": 0.0004999636211999648, "training/grad_norm": 0.309536874294281, "training/positions": 975175680, "training/nonpadding_tokens": 975161320, "throughput/steps_per_second": 0.40821655757780784, "throughput/raw_tokens_per_second": 428035.879664768, "throughput/peak_reserved_gib": 24.353515625}
|
| 150 |
+
{"step": 940, "stop_at": 954, "target": 128747, "training/loss": 5.972557052224874, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 5.972557052224874, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 27.06318424696906, "training/generator_lr": 0.0004999625818844157, "training/grad_norm": 0.3695002794265747, "training/positions": 985661440, "training/nonpadding_tokens": 985647032, "throughput/steps_per_second": 0.40827329895967845, "throughput/raw_tokens_per_second": 428103.6230181088, "throughput/peak_reserved_gib": 24.353515625}
|
| 151 |
+
{"step": 950, "stop_at": 954, "target": 128747, "training/loss": 5.9702926211059095, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 5.9702926211059095, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 27.120598936559364, "training/generator_lr": 0.0004999615279321202, "training/grad_norm": 0.32424482703208923, "training/positions": 996147200, "training/nonpadding_tokens": 996132624, "throughput/steps_per_second": 0.40862160942728804, "throughput/raw_tokens_per_second": 428463.9478837896, "throughput/peak_reserved_gib": 24.353515625}
|
| 152 |
+
{"step": 954, "stop_at": 954, "target": 128747, "training/loss": 5.964567292481661, "training/reconstruction_loss": 0.0, "training/vq_loss": 0.0, "training/ncp_loss": 5.964567292481661, "training/mstok_loss": 0.0, "training/semantic_loss": 0.0, "training/ncp_accuracy": 27.205557424487875, "training/generator_lr": 0.0004999611022529271, "training/grad_norm": 0.36895960569381714, "training/positions": 1000341504, "training/nonpadding_tokens": 1000326925, "throughput/steps_per_second": 0.40841710841631407, "throughput/raw_tokens_per_second": 428256.07156191365, "throughput/peak_reserved_gib": 24.353515625}
|
| 153 |
+
{"validation_step": 954, "loss": 5.56998220205307, "reconstruction_loss": 0.0, "vq_loss": 0.0, "ncp_loss": 5.56998220205307, "mstok_loss": 0.0, "semantic_loss": 0.0, "ncp_accuracy": 32.849665551839465}
|
| 154 |
+
wandb: - 0.022 MB of 0.022 MB uploaded
|
| 155 |
+
wandb: W&B sync reduced upload amount by 4.4%
|
| 156 |
+
wandb:
|
| 157 |
+
wandb: Run summary:
|
| 158 |
+
wandb: throughput/peak_reserved_gib 24.35352
|
| 159 |
+
wandb: throughput/raw_tokens_per_second 428256.07156
|
| 160 |
+
wandb: throughput/steps_per_second 0.40842
|
| 161 |
+
wandb: training/generator_lr 0.0005
|
| 162 |
+
wandb: training/grad_norm 0.36896
|
| 163 |
+
wandb: training/loss 5.96457
|
| 164 |
+
wandb: training/mstok_loss 0.0
|
| 165 |
+
wandb: training/ncp_accuracy 27.20556
|
| 166 |
+
wandb: training/ncp_loss 5.96457
|
| 167 |
+
wandb: training/nonpadding_tokens 1000326925
|
| 168 |
+
wandb: training/positions 1000341504
|
| 169 |
+
wandb: training/reconstruction_loss 0.0
|
| 170 |
+
wandb: training/semantic_loss 0.0
|
| 171 |
+
wandb: training/vq_loss 0.0
|
| 172 |
+
wandb: validation/loss 5.56998
|
| 173 |
+
wandb: validation/mstok_loss 0.0
|
| 174 |
+
wandb: validation/ncp_accuracy 32.84967
|
| 175 |
+
wandb: validation/ncp_loss 5.56998
|
| 176 |
+
wandb: validation/reconstruction_loss 0.0
|
| 177 |
+
wandb: validation/semantic_loss 0.0
|
| 178 |
+
wandb: validation/vq_loss 0.0
|
| 179 |
+
wandb:
|
| 180 |
+
wandb: View run iclr-debug-generator-corruption-uniform-10-135b-v1-generator at: https://wandb.ai/mstok/iclr-debug/runs/iclr-debug-generator-corruption-uniform-10-135b-v1-generator
|
| 181 |
+
wandb: View project at: https://wandb.ai/mstok/iclr-debug
|
| 182 |
+
wandb: Synced 6 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)
|
| 183 |
+
wandb: Find logs at: ./wandb/run-20260919_061426-iclr-debug-generator-corruption-uniform-10-135b-v1-generator/logs
|
| 184 |
+
wandb: WARNING wandb version 0.30.0 is available! To upgrade, please run:
|
| 185 |
+
wandb: WARNING $ pip install wandb --upgrade
|
| 186 |
+
wandb: WARNING The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require("core")`! See https://wandb.me/wandb-core for more information.
|
iclr-debug-generator-corruption-uniform-10-135b-v1/git-provenance.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"commit": "9ec67f19c85e37b60629f945f76b28dbf0e62e65"
|
| 3 |
+
}
|
iclr-debug-generator-corruption-uniform-10-135b-v1/status.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"stage": "stopped-by-user",
|
| 3 |
+
"updated_utc": "2026-09-21T04:57:55.853043+00:00",
|
| 4 |
+
"controller_pid": 36172,
|
| 5 |
+
"child_pid": null,
|
| 6 |
+
"step": 49000,
|
| 7 |
+
"training_stopped": true,
|
| 8 |
+
"evaluation_complete": true,
|
| 9 |
+
"hf_commit": "5dc54d5c52dc23d9e99c06f1b5d73a02a7bc6479",
|
| 10 |
+
"hf_verified_files": 17
|
| 11 |
+
}
|