File size: 4,082 Bytes
99df2d0 5e571cd 99df2d0 5e571cd 99df2d0 5e571cd 99df2d0 5e571cd 99df2d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | {
"schema_version": 2,
"revision": 2,
"revision_id": "r002",
"release": "Pollock 1.0",
"model_id": "SlayerLab/pollock-mini-lm-125m",
"source_checkpoint": {
"path_in_training_workspace": "runs/minimal-en-125m-4ep/ckpt.pt",
"sha256": "026f54a390036b35792aa8fb131c8b0d394efb6d933753fb0e1257e55a67374b",
"iteration": 22004,
"tokens_seen": 10815406080,
"native_nanogpt_parameters": 126637952,
"native_unique_trainable_parameters": 127555456
},
"architecture": {
"n_layer": 12,
"n_head": 14,
"n_embd": 896,
"block_size": 1024,
"vocab_size": 12288,
"dropout": 0.0,
"bias": false,
"tied_word_embeddings": true
},
"training": {
"dataset": "SlayerLab/minimal-en-corpus-2.5b",
"init_from": "scratch",
"micro_batch_per_gpu": 12,
"gradient_accumulation_global": 40,
"ddp_world_size": 2,
"effective_batch_tokens": 491520,
"optimizer": "fused AdamW",
"learning_rate": 0.0003,
"min_learning_rate": 0.00003,
"schedule": "cosine",
"warmup_iters": 440,
"lr_decay_iters": 22003,
"beta1": 0.9,
"beta2": 0.95,
"weight_decay": 0.1,
"grad_clip": 1.0,
"precision": "bfloat16",
"compile": true,
"backend": "nccl",
"seed": 1337,
"hardware": "2x NVIDIA GeForce RTX 4090 24 GB",
"nanogpt_commit": "3adf61e"
},
"evaluation": {
"final_sampled_validation_loss": 2.577547,
"best_sampled_validation_loss": 2.56,
"best_sampled_validation_step": 20000,
"training_eval_batches": 100,
"benchmark_harness": "lm-evaluation-harness 0.4.12",
"benchmark_num_fewshot": 0,
"benchmark_batch_size": 8,
"benchmarks": {
"blimp": {"acc": 0.7669701492537313, "samples": 67000},
"lambada_openai": {"acc": 0.2780904327576169, "perplexity": 53.66752251060021, "samples": 5153},
"hellaswag": {"acc_norm": 0.29874526986656047, "acc": 0.2818163712407887, "samples": 10042},
"piqa": {"acc_norm": 0.6033732317736671, "acc": 0.6137105549510338, "samples": 1838},
"sciq": {"acc_norm": 0.658, "acc": 0.737, "samples": 1000},
"arc_easy": {"acc_norm": 0.42297979797979796, "acc": 0.4659090909090909, "samples": 2376},
"arc_challenge": {"acc_norm": 0.24146757679180889, "acc": 0.20819112627986347, "samples": 1172}
}
},
"conversion": {
"target_class": "GPT2LMHeadModel",
"transformers_version": "5.15.1",
"unique_serialized_parameters": 127674624,
"compatibility_zero_bias_parameters": 119168,
"validation_probe_shape": [2, 64],
"max_absolute_logit_error": 0.0
},
"artifacts": {
"README.md": {"sha256": "789645020a3864ad81e8a5271118aeed322c9a56039209e715689866d781342f"},
"CHANGELOG.md": {"sha256": "379bf0c49f8a8c4a2d40e3ae1775876c37de2e6dac8125385be77742c96529d0"},
"training-history/r001.md": {"sha256": "5e3cb7664fa09902369bb650d1f94cc63b1f0bd1c8a7ffea5c1e6c16c8c4c5df"},
"training-history/r002.md": {"sha256": "bf4364b27d966fe32676d75c269dfd1e53a4939246a0888c0cce7cd63641952d"},
"benchmarks/english.json": {"sha256": "eff82f7d45c82d715c536ac728787ea1eee42515501d4a0544934badab69ce2d"},
"LICENSE.md": {"sha256": "46cbe928ed0aa24875f02f774313b27ec9d8abdf41f0c9ef67e0adb4e0614de4"},
"config.json": {"sha256": "0ad9e47efb8d2ddf4666f016a286604365faa4ff6ece9654718c3e4daa43c41e"},
"generation_config.json": {"sha256": "435beb27be51f0ed054f4a011e5109d125cdadc118b8799b18b155cc798d94d2"},
"model.safetensors": {"sha256": "2aada5b26abe9ce3b70af393d77be4ed1a799a7205b9c0dd5cfaff8bcaabce68"},
"special_tokens_map.json": {"sha256": "8b2257a17ea997bb038f43b133aefec82344ad2b8abc2b8a02a6c0a994ed624e"},
"tokenizer.json": {"sha256": "6cda4e5ec8293f3b821e02253f9c0e88e43ad4f7c6e1324763e1fb91749fef51"},
"tokenizer_config.json": {"sha256": "4cdabe37dbdc1adfcc017ee9a1f86ab89bdf184827d2d9f05181cae0f8af19bf"},
"logs/training.log": {"sha256": "ff856088bdf504759600553a4e9ad5a23f965d54646838adcf0599bb1621d6e1"},
"logs/benchmark-english.log": {"sha256": "f81ad6c915ae49a0d49f61c15ea86a265c31364e5418150c35aa416d4809e22a"}
}
}
|