| { |
| "schema_version": 2, |
| "revision": 2, |
| "revision_id": "r002", |
| "release": "Pollock 1.0", |
| "model_id": "SlayerLab/pollock-mini-lm-125m", |
| "source_checkpoint": { |
| "path_in_training_workspace": "runs/minimal-en-125m-4ep/ckpt.pt", |
| "sha256": "026f54a390036b35792aa8fb131c8b0d394efb6d933753fb0e1257e55a67374b", |
| "iteration": 22004, |
| "tokens_seen": 10815406080, |
| "native_nanogpt_parameters": 126637952, |
| "native_unique_trainable_parameters": 127555456 |
| }, |
| "architecture": { |
| "n_layer": 12, |
| "n_head": 14, |
| "n_embd": 896, |
| "block_size": 1024, |
| "vocab_size": 12288, |
| "dropout": 0.0, |
| "bias": false, |
| "tied_word_embeddings": true |
| }, |
| "training": { |
| "dataset": "SlayerLab/minimal-en-corpus-2.5b", |
| "init_from": "scratch", |
| "micro_batch_per_gpu": 12, |
| "gradient_accumulation_global": 40, |
| "ddp_world_size": 2, |
| "effective_batch_tokens": 491520, |
| "optimizer": "fused AdamW", |
| "learning_rate": 0.0003, |
| "min_learning_rate": 0.00003, |
| "schedule": "cosine", |
| "warmup_iters": 440, |
| "lr_decay_iters": 22003, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "weight_decay": 0.1, |
| "grad_clip": 1.0, |
| "precision": "bfloat16", |
| "compile": true, |
| "backend": "nccl", |
| "seed": 1337, |
| "hardware": "2x NVIDIA GeForce RTX 4090 24 GB", |
| "nanogpt_commit": "3adf61e" |
| }, |
| "evaluation": { |
| "final_sampled_validation_loss": 2.577547, |
| "best_sampled_validation_loss": 2.56, |
| "best_sampled_validation_step": 20000, |
| "training_eval_batches": 100, |
| "benchmark_harness": "lm-evaluation-harness 0.4.12", |
| "benchmark_num_fewshot": 0, |
| "benchmark_batch_size": 8, |
| "benchmarks": { |
| "blimp": {"acc": 0.7669701492537313, "samples": 67000}, |
| "lambada_openai": {"acc": 0.2780904327576169, "perplexity": 53.66752251060021, "samples": 5153}, |
| "hellaswag": {"acc_norm": 0.29874526986656047, "acc": 0.2818163712407887, "samples": 10042}, |
| "piqa": {"acc_norm": 0.6033732317736671, "acc": 0.6137105549510338, "samples": 1838}, |
| "sciq": {"acc_norm": 0.658, "acc": 0.737, "samples": 1000}, |
| "arc_easy": {"acc_norm": 0.42297979797979796, "acc": 0.4659090909090909, "samples": 2376}, |
| "arc_challenge": {"acc_norm": 0.24146757679180889, "acc": 0.20819112627986347, "samples": 1172} |
| } |
| }, |
| "conversion": { |
| "target_class": "GPT2LMHeadModel", |
| "transformers_version": "5.15.1", |
| "unique_serialized_parameters": 127674624, |
| "compatibility_zero_bias_parameters": 119168, |
| "validation_probe_shape": [2, 64], |
| "max_absolute_logit_error": 0.0 |
| }, |
| "artifacts": { |
| "README.md": {"sha256": "789645020a3864ad81e8a5271118aeed322c9a56039209e715689866d781342f"}, |
| "CHANGELOG.md": {"sha256": "379bf0c49f8a8c4a2d40e3ae1775876c37de2e6dac8125385be77742c96529d0"}, |
| "training-history/r001.md": {"sha256": "5e3cb7664fa09902369bb650d1f94cc63b1f0bd1c8a7ffea5c1e6c16c8c4c5df"}, |
| "training-history/r002.md": {"sha256": "bf4364b27d966fe32676d75c269dfd1e53a4939246a0888c0cce7cd63641952d"}, |
| "benchmarks/english.json": {"sha256": "eff82f7d45c82d715c536ac728787ea1eee42515501d4a0544934badab69ce2d"}, |
| "LICENSE.md": {"sha256": "46cbe928ed0aa24875f02f774313b27ec9d8abdf41f0c9ef67e0adb4e0614de4"}, |
| "config.json": {"sha256": "0ad9e47efb8d2ddf4666f016a286604365faa4ff6ece9654718c3e4daa43c41e"}, |
| "generation_config.json": {"sha256": "435beb27be51f0ed054f4a011e5109d125cdadc118b8799b18b155cc798d94d2"}, |
| "model.safetensors": {"sha256": "2aada5b26abe9ce3b70af393d77be4ed1a799a7205b9c0dd5cfaff8bcaabce68"}, |
| "special_tokens_map.json": {"sha256": "8b2257a17ea997bb038f43b133aefec82344ad2b8abc2b8a02a6c0a994ed624e"}, |
| "tokenizer.json": {"sha256": "6cda4e5ec8293f3b821e02253f9c0e88e43ad4f7c6e1324763e1fb91749fef51"}, |
| "tokenizer_config.json": {"sha256": "4cdabe37dbdc1adfcc017ee9a1f86ab89bdf184827d2d9f05181cae0f8af19bf"}, |
| "logs/training.log": {"sha256": "ff856088bdf504759600553a4e9ad5a23f965d54646838adcf0599bb1621d6e1"}, |
| "logs/benchmark-english.log": {"sha256": "f81ad6c915ae49a0d49f61c15ea86a265c31364e5418150c35aa416d4809e22a"} |
| } |
| } |
|
|
