diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1/summary.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a757a8d3f794cc2971d9b336ff29198b06062e77 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1/summary.json @@ -0,0 +1,25 @@ +{ + "experiment_id": "Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1", + "stage": "sft", + "base_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_checkpoint_step": 9600, + "step": 160, + "stage_training_flops": 3.8505838661546803e+18, + "inherited_parent_flops": 2.3103503196928082e+20, + "cumulative_pipeline_training_flops": 2.348856158354355e+20, + "chatcore_metric": 0.13532147122389065, + "chat_results": { + "ARC-Easy": 0.4553872053872054, + "ARC-Challenge": 0.378839590443686, + "MMLU": 0.3474576271186441, + "GSM8K": 0.06444275966641395, + "HumanEval": 0.036585365853658534 + }, + "training_time_seconds": 6951.126858472824, + "config_fingerprint": "6359be7150ece075", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "3c34775d260a016a79ab46ea76f8c8e48458fd68", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/8c553682", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1" +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/eval_metrics.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/eval_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f3e7eb210d99f103c00dafd9da0415b5c1dc72c7 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/eval_metrics.json @@ -0,0 +1,31 @@ +{ + "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2", + "recipe": "curriculum", + "step": 42, + "val_bpb": 0.7037352212342954, + "min_val_bpb": 0.7037352212342954, + "per_route_bpb": {}, + "per_domain_bpb": {}, + "curriculum_summary": null, + "chatcore": { + "chatcore_metric": -0.011321608241655831, + "chatcore_cat": -0.018869347069426386, + "suite": { + "name": "karpathy", + "tasks": [ + "ARC-Easy", + "ARC-Challenge", + "MMLU", + "GSM8K", + "HumanEval" + ], + "max_generative_problems": null, + "generative_answer_format": null + }, + "ARC-Easy": 0.24579124579124578, + "ARC-Challenge": 0.23037542662116042, + "MMLU": 0.23137729668138443, + "GSM8K": 0.0, + "HumanEval": 0.0 + } +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/meta_000042.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/meta_000042.json new file mode 100644 index 0000000000000000000000000000000000000000..d85126cc828cf0b36d544c33a4d4f59016e00897 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/meta_000042.json @@ -0,0 +1,178 @@ +{ + "step": 42, + "training_complete": true, + "val_bpb": 0.7037352212342954, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2", + "wandb_run_id": "036b1c71", + "wandb_group": "think-d32", + "wandb_tags": "sft,curriculum,c3,scale-max,staged,robustness,noise,d32", + "device_type": "", + "model_tag": null, + "model_step": null, + "base_checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/base_checkpoints", + "base_step": 9600, + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer", + "resume_from_step": null, + "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/config.json", + "parent_cumulative_flops": 2.3103503196928082e+20, + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "04bb043ad61d4db3e9022c422302df7b8f2dc0c9", + "load_optimizer": 0, + "num_iterations": -1, + "max_seq_len": null, + "device_batch_size": 2, + "total_batch_size": null, + "embedding_lr": null, + "unembedding_lr": null, + "matrix_lr": null, + "init_lr_frac": 0.8, + "warmup_ratio": 0.03, + "warmdown_ratio": 0.5, + "final_lr_frac": 0.0, + "eval_every": 400, + "eval_tokens": 20971520, + "chatcore_every": 100000, + "chatcore_max_cat": -1, + "chatcore_max_sample": 32, + "save_every": -1, + "recipe": "curriculum", + "curriculum_config": "", + "pre1930_epochs": 5, + "mmlu_epochs": 3, + "gsm8k_epochs": 4, + "authentic_epochs": 0, + "knowledge_qa_epochs": 0, + "multiturn_qa_epochs": 0, + "reasoning_qa_epochs": 0, + "stem_reasoning_epochs": 0, + "narrative_grounded_epochs": 0, + "narrative_fiction_epochs": 0, + "opinion_qa_epochs": 0, + "how_to_qa_epochs": 0, + "verse_qa_epochs": 0, + "composition_qa_epochs": 0, + "resolved_experiment_config": { + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-curriculum-c3-robust-v2", + "data": { + "recipe": "curriculum", + "curriculum": { + "name": "C3Rv2", + "mode": "staged", + "threshold_default": 80, + "stages": [ + { + "routes": [ + "knowledge_qa" + ], + "authentic": "single" + }, + { + "routes": [ + "reasoning_qa", + "stem_reasoning", + "how_to_qa", + "opinion_qa", + "composition_qa", + "verse_qa" + ], + "calibration_qa": true + }, + { + "routes": [ + "multiturn_qa", + "narrative_grounded", + "narrative_fiction" + ], + "authentic": "multi" + } + ], + "noise": { + "rate": 0.3 + }, + "robustness": { + "stage": 0, + "epochs": 1, + "routes": { + "conversation_qa": { + "count": null + }, + "unparseable_qa": { + "count": null + }, + "typo_qa": { + "count": null + }, + "era_qa": { + "count": null + }, + "conversation_multiturn": { + "count": null + } + } + } + } + }, + "training": { + "num_iterations": -1, + "load_optimizer": 0, + "device_batch_size": 2, + "warmup_ratio": 0.03, + "eval_every": 400, + "chatcore_every": 100000, + "chatcore_max_sample": 32, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d32", + "tags": [ + "sft", + "curriculum", + "c3", + "scale-max", + "staged", + "robustness", + "noise", + "d32" + ] + }, + "config_fingerprint": "4462a389dc98cd5d", + "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2" + }, + "stage": "sft", + "base_experiment_id": null, + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "4462a389dc98cd5d" + }, + "loop_state": { + "step": 42, + "total_training_time": 1503.5729036331177, + "min_val_bpb": 0.7037352212342954, + "smooth_train_loss": 1.8593226867746961, + "mfu": 51.82901425419851, + "tok_per_sec": 44667, + "stage_training_flops": 1.0107782648656036e+18, + "inherited_parent_flops": 2.3103503196928082e+20, + "cumulative_pipeline_training_flops": 2.3204581023414642e+20 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/model_000042.pt b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/model_000042.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c551ed82c9b4b1612a631df6e091a98d2dcbad5 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/model_000042.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a9bc83e3853f0a7ad8b27cc45388d727cc5976837f514d61ea59f80d7f0348 +size 8992694653 diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/optim_000042_rank0.pt b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/optim_000042_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ba76a7fff00118ac728cfbc451ad5a80e034586 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/checkpoints/optim_000042_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34c0871ec1aed5ced55a463ea1b827e4ef80e286735cc42512ced11f8c579fe4 +size 11545903753 diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/config.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e02c2e96ea39a561695c7897f006221a2c6bcbb --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/config.json @@ -0,0 +1,95 @@ +{ + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-curriculum-c3-robust-v2", + "data": { + "recipe": "curriculum", + "curriculum": { + "name": "C3Rv2", + "mode": "staged", + "threshold_default": 80, + "stages": [ + { + "routes": [ + "knowledge_qa" + ], + "authentic": "single" + }, + { + "routes": [ + "reasoning_qa", + "stem_reasoning", + "how_to_qa", + "opinion_qa", + "composition_qa", + "verse_qa" + ], + "calibration_qa": true + }, + { + "routes": [ + "multiturn_qa", + "narrative_grounded", + "narrative_fiction" + ], + "authentic": "multi" + } + ], + "noise": { + "rate": 0.3 + }, + "robustness": { + "stage": 0, + "epochs": 1, + "routes": { + "conversation_qa": { + "count": null + }, + "unparseable_qa": { + "count": null + }, + "typo_qa": { + "count": null + }, + "era_qa": { + "count": null + }, + "conversation_multiturn": { + "count": null + } + } + } + } + }, + "training": { + "num_iterations": -1, + "load_optimizer": 0, + "device_batch_size": 2, + "warmup_ratio": 0.03, + "eval_every": 400, + "chatcore_every": 100000, + "chatcore_max_sample": 32, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d32", + "tags": [ + "sft", + "curriculum", + "c3", + "scale-max", + "staged", + "robustness", + "noise", + "d32" + ] + }, + "config_fingerprint": "4462a389dc98cd5d", + "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2" +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/evals/chatcore.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/evals/chatcore.json new file mode 100644 index 0000000000000000000000000000000000000000..22aaa6fbaa4fafd91f83241a19976f553ed72519 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/evals/chatcore.json @@ -0,0 +1,29 @@ +{ + "stage": "sft", + "step": 42, + "total_training_flops": 2.3204581023414642e+20, + "stage_training_flops": 1.0107782648656036e+18, + "inherited_parent_flops": 2.3103503196928082e+20, + "cumulative_pipeline_training_flops": 2.3204581023414642e+20, + "results": { + "ARC-Easy": 0.24579124579124578, + "ARC-Challenge": 0.23037542662116042, + "MMLU": 0.23137729668138443, + "GSM8K": 0.0, + "HumanEval": 0.0 + }, + "chatcore_metric": -0.011321608241655831, + "chatcore_suite": { + "name": "karpathy", + "tasks": [ + "ARC-Easy", + "ARC-Challenge", + "MMLU", + "GSM8K", + "HumanEval" + ], + "max_generative_problems": null, + "generative_answer_format": null + }, + "complete": true +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/run.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/run.json new file mode 100644 index 0000000000000000000000000000000000000000..f5bb2a33604ebc019414f11497210eca388f8274 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v2", + "stage": "sft", + "base_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_checkpoint_step": 9600, + "branch_parent_step": null, + "config_fingerprint": "4462a389dc98cd5d", + "wandb_run_id": "036b1c71", + "created_at": 1786715992 +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/config.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a4ddf2ba6dc539aaf0dc31a8a6b692b20426b89a --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/config.json @@ -0,0 +1,97 @@ +{ + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-curriculum-c3-robust-v3-a100-40gb", + "data": { + "recipe": "curriculum", + "curriculum": { + "name": "C3Rv3", + "mode": "staged", + "passes": 3, + "threshold_default": 80, + "stages": [ + { + "routes": [ + "knowledge_qa" + ], + "authentic": "single" + }, + { + "routes": [ + "reasoning_qa", + "stem_reasoning", + "how_to_qa", + "opinion_qa", + "composition_qa", + "verse_qa" + ], + "calibration_qa": true + }, + { + "routes": [ + "multiturn_qa", + "narrative_grounded", + "narrative_fiction" + ], + "authentic": "multi" + } + ], + "noise": { + "rate": 0.3, + "end_punct_rate": 0.05 + }, + "robustness": { + "stage": 0, + "epochs": 2.5, + "routes": { + "conversation_qa": { + "count": null + }, + "unparseable_qa": { + "count": null + }, + "typo_qa": { + "count": null + }, + "era_qa": { + "count": null + }, + "conversation_multiturn": { + "count": null + } + } + } + } + }, + "training": { + "num_iterations": -1, + "load_optimizer": 0, + "device_batch_size": 1, + "warmup_ratio": 0.03, + "eval_every": 400, + "chatcore_every": 100000, + "chatcore_max_sample": 32, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d32", + "tags": [ + "sft", + "curriculum", + "c3", + "scale-max", + "staged", + "robustness", + "noise", + "d32" + ] + }, + "config_fingerprint": "b0d39eef6f8e427b", + "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb" +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/run.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/run.json new file mode 100644 index 0000000000000000000000000000000000000000..7e26290a20db3dbd2a963181d46a4707e7763913 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3-a100-40gb", + "stage": "sft", + "base_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_checkpoint_step": 9600, + "branch_parent_step": null, + "config_fingerprint": "b0d39eef6f8e427b", + "wandb_run_id": "5667c0f5", + "created_at": 1787104196 +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/config.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8c521169515b02fe0eeb96ee888c1344007cb256 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/config.json @@ -0,0 +1,97 @@ +{ + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-curriculum-c3-robust-v3", + "data": { + "recipe": "curriculum", + "curriculum": { + "name": "C3Rv3", + "mode": "staged", + "passes": 3, + "threshold_default": 80, + "stages": [ + { + "routes": [ + "knowledge_qa" + ], + "authentic": "single" + }, + { + "routes": [ + "reasoning_qa", + "stem_reasoning", + "how_to_qa", + "opinion_qa", + "composition_qa", + "verse_qa" + ], + "calibration_qa": true + }, + { + "routes": [ + "multiturn_qa", + "narrative_grounded", + "narrative_fiction" + ], + "authentic": "multi" + } + ], + "noise": { + "rate": 0.3, + "end_punct_rate": 0.05 + }, + "robustness": { + "stage": 0, + "epochs": 2.5, + "routes": { + "conversation_qa": { + "count": null + }, + "unparseable_qa": { + "count": null + }, + "typo_qa": { + "count": null + }, + "era_qa": { + "count": null + }, + "conversation_multiturn": { + "count": null + } + } + } + } + }, + "training": { + "num_iterations": -1, + "load_optimizer": 0, + "device_batch_size": 2, + "warmup_ratio": 0.03, + "eval_every": 400, + "chatcore_every": 100000, + "chatcore_max_sample": 32, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d32", + "tags": [ + "sft", + "curriculum", + "c3", + "scale-max", + "staged", + "robustness", + "noise", + "d32" + ] + }, + "config_fingerprint": "12ffd193bdca635a", + "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3" +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/run.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/run.json new file mode 100644 index 0000000000000000000000000000000000000000..49f8abf4bab7661292b549f3d45f760f967fdec0 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust-v3", + "stage": "sft", + "base_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_checkpoint_step": 9600, + "branch_parent_step": null, + "config_fingerprint": "12ffd193bdca635a", + "wandb_run_id": "10993832", + "created_at": 1787101675 +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/meta_000041.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/meta_000041.json new file mode 100644 index 0000000000000000000000000000000000000000..cf590f0bd077db8b95de142b78e85230c742e9ec --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/meta_000041.json @@ -0,0 +1,172 @@ +{ + "step": 41, + "training_complete": true, + "val_bpb": 0.7046443464143689, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust", + "wandb_run_id": "21793571", + "wandb_group": "think-d32", + "wandb_tags": "sft,curriculum,c3,scale-max,staged,robustness,noise,d32", + "device_type": "", + "model_tag": null, + "model_step": null, + "base_checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/base_checkpoints", + "base_step": 9600, + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer", + "resume_from_step": null, + "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/config.json", + "parent_cumulative_flops": 2.3103503196928082e+20, + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "8549a37b028418b0abaf347bd6e5a8ccb6cff782", + "load_optimizer": 0, + "num_iterations": -1, + "max_seq_len": null, + "device_batch_size": 2, + "total_batch_size": null, + "embedding_lr": null, + "unembedding_lr": null, + "matrix_lr": null, + "init_lr_frac": 0.8, + "warmup_ratio": 0.03, + "warmdown_ratio": 0.5, + "final_lr_frac": 0.0, + "eval_every": 400, + "eval_tokens": 20971520, + "chatcore_every": 100000, + "chatcore_max_cat": -1, + "chatcore_max_sample": 32, + "save_every": -1, + "recipe": "curriculum", + "curriculum_config": "", + "pre1930_epochs": 5, + "mmlu_epochs": 3, + "gsm8k_epochs": 4, + "authentic_epochs": 0, + "knowledge_qa_epochs": 0, + "multiturn_qa_epochs": 0, + "reasoning_qa_epochs": 0, + "stem_reasoning_epochs": 0, + "narrative_grounded_epochs": 0, + "narrative_fiction_epochs": 0, + "opinion_qa_epochs": 0, + "how_to_qa_epochs": 0, + "verse_qa_epochs": 0, + "composition_qa_epochs": 0, + "resolved_experiment_config": { + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-curriculum-c3-robust", + "data": { + "recipe": "curriculum", + "curriculum": { + "name": "C3R", + "mode": "staged", + "threshold_default": 80, + "stages": [ + { + "routes": [ + "knowledge_qa" + ], + "authentic": "single" + }, + { + "routes": [ + "reasoning_qa", + "stem_reasoning", + "how_to_qa", + "opinion_qa", + "composition_qa", + "verse_qa" + ], + "calibration_qa": true + }, + { + "routes": [ + "multiturn_qa", + "narrative_grounded", + "narrative_fiction" + ], + "authentic": "multi" + } + ], + "noise": { + "rate": 0.3 + }, + "robustness": { + "stage": 0, + "epochs": 1, + "routes": { + "conversation_qa": { + "count": null + }, + "unparseable_qa": { + "count": null + }, + "era_qa": { + "count": null + } + } + } + } + }, + "training": { + "num_iterations": -1, + "load_optimizer": 0, + "device_batch_size": 2, + "warmup_ratio": 0.03, + "eval_every": 400, + "chatcore_every": 100000, + "chatcore_max_sample": 32, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d32", + "tags": [ + "sft", + "curriculum", + "c3", + "scale-max", + "staged", + "robustness", + "noise", + "d32" + ] + }, + "config_fingerprint": "e1a9d25fb1cf5edf", + "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust" + }, + "stage": "sft", + "base_experiment_id": null, + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "e1a9d25fb1cf5edf" + }, + "loop_state": { + "step": 41, + "total_training_time": 1459.9074046611786, + "min_val_bpb": 0.7046443464143689, + "smooth_train_loss": 1.907550057536094, + "mfu": 51.698687069018604, + "tok_per_sec": 44555, + "stage_training_flops": 9.867121157021368e+17, + "inherited_parent_flops": 2.3103503196928082e+20, + "cumulative_pipeline_training_flops": 2.3202174408498296e+20 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/model_000041.pt b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/model_000041.pt new file mode 100644 index 0000000000000000000000000000000000000000..99714f4aba77f97c5899d2f390de263ff59385f4 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/model_000041.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:590b64a7ab561ae7ec7adf10fd09469e7b46dacf76e5c665f743fd9d0dd74a6f +size 8992694653 diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/optim_000041_rank0.pt b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/optim_000041_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..16a2b5c288bb24620b6506caef0127d5e405cce0 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/checkpoints/optim_000041_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c642c3e04109fce332ca03b7afaf571812e0d9ac124f8863889f262a6748995 +size 11545903753 diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/config.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/config.json new file mode 100644 index 0000000000000000000000000000000000000000..489161da052f062973e9fa79d3a4ed90fc733957 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/config.json @@ -0,0 +1,89 @@ +{ + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-curriculum-c3-robust", + "data": { + "recipe": "curriculum", + "curriculum": { + "name": "C3R", + "mode": "staged", + "threshold_default": 80, + "stages": [ + { + "routes": [ + "knowledge_qa" + ], + "authentic": "single" + }, + { + "routes": [ + "reasoning_qa", + "stem_reasoning", + "how_to_qa", + "opinion_qa", + "composition_qa", + "verse_qa" + ], + "calibration_qa": true + }, + { + "routes": [ + "multiturn_qa", + "narrative_grounded", + "narrative_fiction" + ], + "authentic": "multi" + } + ], + "noise": { + "rate": 0.3 + }, + "robustness": { + "stage": 0, + "epochs": 1, + "routes": { + "conversation_qa": { + "count": null + }, + "unparseable_qa": { + "count": null + }, + "era_qa": { + "count": null + } + } + } + } + }, + "training": { + "num_iterations": -1, + "load_optimizer": 0, + "device_batch_size": 2, + "warmup_ratio": 0.03, + "eval_every": 400, + "chatcore_every": 100000, + "chatcore_max_sample": 32, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d32", + "tags": [ + "sft", + "curriculum", + "c3", + "scale-max", + "staged", + "robustness", + "noise", + "d32" + ] + }, + "config_fingerprint": "e1a9d25fb1cf5edf", + "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust" +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/run.json b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/run.json new file mode 100644 index 0000000000000000000000000000000000000000..702fe685f0d2c9a3a50218e829b938ac8b8e4032 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "Think.Unbounded-d32-v2mix-cont-pre1930-curriculum-c3-robust", + "stage": "sft", + "base_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_checkpoint_step": 9600, + "branch_parent_step": null, + "config_fingerprint": "e1a9d25fb1cf5edf", + "wandb_run_id": "21793571", + "created_at": 1786635058 +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/summary.json b/experiments/Think.Unbounded-d32-v2mix-cont/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bab0e720b7ef299b29a99cb887fa99992d12b983 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/summary.json @@ -0,0 +1,79 @@ +{ + "experiment_id": "Think.Unbounded-d32-v2mix-cont", + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32-v2mix-cont", + "parent_experiment_id": "Think.Unbounded-d32", + "parent_checkpoint_step": 5500, + "dataset": "jbduran/think-dataset-clean-1930s", + "dataset_revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "step": 9600, + "depth": 32, + "target_param_data_ratio": 12, + "training_tokens": 20132659200, + "final_sampled_val_bpb": 0.7708812390249374, + "minimum_sampled_val_bpb": 0.7701763237939314, + "full_val_bpb": 0.7505380497637221, + "core_metric": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the most important city in the world. It is the capital of the most powerful" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is Au, and that of silver is Ag. The symbol of the former is Ag" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If yesterday was Saturday, then tomorrow will be Sunday. If yesterday was" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is cold, and the opposite of cold is hot. The opposite of hot is cold" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: Mercury, Venus, Mars, Jupiter, Saturn, and Herschel. The earth" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a deep, rich, velvety crimson, the same shade as the rose" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is 5.\n\nIf 3x + 4 = 13, then x" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay originated with Mr. W. C. Barrett, who says of it: \"Mr. Galton's 'Life and Times,' and his Life of Darwin.' \"It is mainly a half-tone reprint from Galton's (vice supra) life, to which I have added some important facts which I had not been able to procure when the original edition was published, and which bear directly upon some matters of interest.\n\nIt may, however, be as well to draw attention more particularly to the point to which Mr. Barrett refers in the above quotation, as this contains, I believe, the fundamental distinction between the Darwin", + "<|bos|>Lady Saumarez -- Scotch rose shaded white. Pheasant's Eye -- Pure white, dark eye. Marigold -- Striking light golden color. Maddereti -- A most desirable purple.\n\nRose -- -A superb full flowering rose of a most beautiful shade of deep rose. Sunrise -- Pure tints of fawn and yellow.\n\nCarnation Marguerite\n\nLarge Flowered\n\nnow offered. Much hardier and blooms more abundantly.\n\nPrice, any variety, 10 cts. each; set of 1 3 for 45 cents.\n\nEasily Makes a Good Center\n\nAnywhere.\n\n", + "<|bos|>RIDGEN, TOLLAND\n\nFRIDLAND, Dyfe, y 1665.\n\nTHE CONFUCTIONS\n\nor\n\nTHE KINGDOM OF SOLOMON.\n\nTHE FIRST BOOK OF THE KING OF SOLOMON\n\nRefers to the Four Series of the Four Dispensations, God revealing himself to His Children in the various Births of His Israel. The Messiah's birth and baptism of John, the mission of the\n\nApostles, and the baptism of Christ, and latterly the majestic progress of His power, are chronicled in the Books of Moses, of the Prophets, of the Proph ets, and the", + "<|bos|>agers have already told us, and M. Crookes has already informed us, that he found concretions of this nature in the plantlice when giving a supplementary examination to them : hence, even according to his own showing, no safe conclusion can be drawn from the relative order and distribution of these silica-precipitates. At the same time it is natural to ask whether this tardy means of elucidation, instead of becoming antiquated by the lapse of the fair period which Hibbert has now completed-and well-nigh brought to a close by a misuse of some words, may not be pressed into use - --\n\nON", + "<|bos|>D\n\nio\n\nLe\n\nthered in the middle sept i post- ater bear ried ne pro avera hing tenanted\n\nCrowne he se ouch h find\n\n\u200bLa thi seeks depone \u0f51 \u0f4f \u0f63\u0f74 \u0f49\u0f7a\u0f51 \u0f61 \u0f61 \u0f61\u0f72 \u0f64\u0f74 \u0f62 \u0f61 \u0f51 \u0f62 \u3002 \u0f61\u0f72\u0970\u0f62 \u0f51 \u0f0b \u0f62\u0f66 Atth Id\n\nDedication of the Picture in the Library of Trinity that has the most frequently called forth criticism\n\nI am unable to", + "<|bos|>PREFACE\n\nCONTENTS.\n\n1. X. About a month was being wasted at the hotel, when a kindhearted friend offered to go on ahead and engage rooms for us at the best houses the day was hot-the Southern hotels always are-on the other side, and on most of them, indeed, fair profit can be made; and the diligence kept always waiting at the station and often not in readiness for an hour after the last customer had been put in, and the engineer grumbled fearfully at this irregularity of the train, because the trains gave him a pretty lively time when there was anybody waiting there to see, and the porters sometimes began", + "<|bos|>Clarke Model\n\n222000908000 DIANER DER\n\nThe proper unture dishes\n\nWilder Damases7", + "<|bos|>ossoer 0 020 oO 000000 ee OP atte eT es ee eee en eee Bee Se SERRE Nee errre reser Res eererr ee eee cerr reeer era Nee essen re ers 2008 OO OO DO ES OO ESE HD TTI MT SPEC IC PRT APOLOC H OCDOETTES 50 Ce a OC ee DDg ee CF EIOLUS) oe eA tt } 500 een eeehey n" + ], + "training_time_seconds": 164042.2579112053, + "stage_training_flops": 9.867121157021368e+19, + "inherited_parent_flops": 1.3236382039906714e+20, + "cumulative_pipeline_training_flops": 2.3103503196928082e+20, + "config_fingerprint": "00af0bde578c1269", + "git_commit_sha": "734396fce10c8fcd7bf979261d41ebe37968089c", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/2fa1b744", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/Think.Unbounded-d32-v2mix-cont", + "dataset_fingerprint": "54cc47b13796e411", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "unique_train_tokens_per_source": { + "original": 14515647284, + "midtrain_r21": 4107062477, + "midtrain_r45": 2053531239 + }, + "branch_parent_experiment_id": "Think.Unbounded-d32", + "branch_parent_step": 5500, + "branch_lr_schedule": "continue", + "stage_training_tokens": 8598323200, + "unique_train_tokens": 20676241000, + "effective_epochs": 0.41585524177242855 +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/experiment_tokenizer.json b/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78 --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 200, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 1000000000000, + "doc_cap": 1000000000, + "vocab_size": 32768 + }, + "created_at": 1784129374 +} diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/token_bytes.pt b/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2 +size 132649 diff --git a/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/tokenizer.pkl b/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac --- /dev/null +++ b/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f +size 410542 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_000500.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..7935dcc7dfb651299c72419dd22853c0209a9570 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_000500.json @@ -0,0 +1,236 @@ +{ + "step": 500, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.1476923999299866, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48712193, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48712193, + "mixture": { + "cursors": { + "original": { + "file_idx": 10, + "pos": 48712193, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48712193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 1048584192, + "source_tokens": { + "original": 1048584192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 1.1476923999299866, + "smooth_train_loss": 3.427469208772481, + "total_training_time": 19713.592538118362, + "stage_start_step": 0, + "stage_training_flops": 12033074581733376000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.2033074581733376e+19 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_001000.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..219c4ab8436d17235fa15e955c056b199bad077e --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_001000.json @@ -0,0 +1,236 @@ +{ + "step": 1000, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.056581441012716, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 20, + "pos": 97416193, + "epoch": 1, + "pq_idx": 20, + "rg_idx": 97416193, + "mixture": { + "cursors": { + "original": { + "file_idx": 20, + "pos": 97416193, + "epoch": 1, + "pq_idx": 20, + "rg_idx": 97416193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 2097160192, + "source_tokens": { + "original": 2097160192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 1.056581441012716, + "smooth_train_loss": 2.9296221567382985, + "total_training_time": 39794.540657281876, + "stage_start_step": 0, + "stage_training_flops": 24066149163466752000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2.406614916346675e+19 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_001500.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..0f1ef0a6c9588eacb2fa8c3569d9d4a85ddad695 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_001500.json @@ -0,0 +1,236 @@ +{ + "step": 1500, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.0152464516907762, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 31, + "pos": 46120193, + "epoch": 1, + "pq_idx": 31, + "rg_idx": 46120193, + "mixture": { + "cursors": { + "original": { + "file_idx": 31, + "pos": 46120193, + "epoch": 1, + "pq_idx": 31, + "rg_idx": 46120193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 3145736192, + "source_tokens": { + "original": 3145736192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 1.0152464516907762, + "smooth_train_loss": 2.935989382480182, + "total_training_time": 59860.37234377861, + "stage_start_step": 0, + "stage_training_flops": 36099223745200128000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 3.609922374520013e+19 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_002000.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..f2a7acb0803feb91d39817fcd45a4a04b41b5375 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_002000.json @@ -0,0 +1,236 @@ +{ + "step": 2000, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 0.9928518912323213, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 41, + "pos": 94824193, + "epoch": 1, + "pq_idx": 41, + "rg_idx": 94824193, + "mixture": { + "cursors": { + "original": { + "file_idx": 41, + "pos": 94824193, + "epoch": 1, + "pq_idx": 41, + "rg_idx": 94824193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 4194312192, + "source_tokens": { + "original": 4194312192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 0.9928518912323213, + "smooth_train_loss": 2.715263738468478, + "total_training_time": 79915.32717370987, + "stage_start_step": 0, + "stage_training_flops": 48132298326933504000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 4.81322983269335e+19 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_002500.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_002500.json new file mode 100644 index 0000000000000000000000000000000000000000..fc94ea366818b97bfde1f6f3cf5da6c55565bdb2 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_002500.json @@ -0,0 +1,236 @@ +{ + "step": 2500, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 0.9737910804428661, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 52, + "pos": 43528193, + "epoch": 1, + "pq_idx": 52, + "rg_idx": 43528193, + "mixture": { + "cursors": { + "original": { + "file_idx": 52, + "pos": 43528193, + "epoch": 1, + "pq_idx": 52, + "rg_idx": 43528193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 5242888192, + "source_tokens": { + "original": 5242888192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 0.9630578341851044, + "smooth_train_loss": 2.7360114404103504, + "total_training_time": 99964.06782126427, + "stage_start_step": 0, + "stage_training_flops": 60165372908666880000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 6.016537290866688e+19 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_003000.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_003000.json new file mode 100644 index 0000000000000000000000000000000000000000..b8a81d5b057170634b33245f418ef0a6a73a246e --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_003000.json @@ -0,0 +1,236 @@ +{ + "step": 3000, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 0.9735215125930343, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 62, + "pos": 92232193, + "epoch": 1, + "pq_idx": 62, + "rg_idx": 92232193, + "mixture": { + "cursors": { + "original": { + "file_idx": 62, + "pos": 92232193, + "epoch": 1, + "pq_idx": 62, + "rg_idx": 92232193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 6291464192, + "source_tokens": { + "original": 6291464192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 0.9630578341851044, + "smooth_train_loss": 2.6967057059495345, + "total_training_time": 120021.50824666023, + "stage_start_step": 0, + "stage_training_flops": 72198447490400256000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 7.219844749040026e+19 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_003500.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_003500.json new file mode 100644 index 0000000000000000000000000000000000000000..8e770ca7582b046b206cc7188a4512c19d2b6b6a --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_003500.json @@ -0,0 +1,236 @@ +{ + "step": 3500, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 0.968207957699399, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 73, + "pos": 40936193, + "epoch": 1, + "pq_idx": 73, + "rg_idx": 40936193, + "mixture": { + "cursors": { + "original": { + "file_idx": 73, + "pos": 40936193, + "epoch": 1, + "pq_idx": 73, + "rg_idx": 40936193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 7340040192, + "source_tokens": { + "original": 7340040192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 0.9630578341851044, + "smooth_train_loss": 2.5911028486222323, + "total_training_time": 140049.3800020218, + "stage_start_step": 0, + "stage_training_flops": 84231522072133632000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 8.423152207213363e+19 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_004000.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_004000.json new file mode 100644 index 0000000000000000000000000000000000000000..29c848cb860c77857b9d61e09ca9036b75102b08 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_004000.json @@ -0,0 +1,236 @@ +{ + "step": 4000, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 0.9585141600611977, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 83, + "pos": 89640193, + "epoch": 1, + "pq_idx": 83, + "rg_idx": 89640193, + "mixture": { + "cursors": { + "original": { + "file_idx": 83, + "pos": 89640193, + "epoch": 1, + "pq_idx": 83, + "rg_idx": 89640193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 8388616192, + "source_tokens": { + "original": 8388616192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 0.9585141600611977, + "smooth_train_loss": 2.7874954028564223, + "total_training_time": 160072.53116488457, + "stage_start_step": 0, + "stage_training_flops": 96264596653867008000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 9.6264596653867e+19 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_004500.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_004500.json new file mode 100644 index 0000000000000000000000000000000000000000..17dc68edc7e4507427eb4d45adb1098c37e42880 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_004500.json @@ -0,0 +1,236 @@ +{ + "step": 4500, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 0.956614818178498, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 94, + "pos": 38344193, + "epoch": 1, + "pq_idx": 94, + "rg_idx": 38344193, + "mixture": { + "cursors": { + "original": { + "file_idx": 94, + "pos": 38344193, + "epoch": 1, + "pq_idx": 94, + "rg_idx": 38344193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 9437192192, + "source_tokens": { + "original": 9437192192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 0.956614818178498, + "smooth_train_loss": 2.591393936107061, + "total_training_time": 180101.40045905113, + "stage_start_step": 0, + "stage_training_flops": 108297671235600384000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.0829767123560038e+20 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_005000.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_005000.json new file mode 100644 index 0000000000000000000000000000000000000000..09794af0eec806aff048c4816ae11da7f4fdc682 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_005000.json @@ -0,0 +1,236 @@ +{ + "step": 5000, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 0.9363505794880082, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 104, + "pos": 87048193, + "epoch": 1, + "pq_idx": 104, + "rg_idx": 87048193, + "mixture": { + "cursors": { + "original": { + "file_idx": 104, + "pos": 87048193, + "epoch": 1, + "pq_idx": 104, + "rg_idx": 87048193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 10485768192, + "source_tokens": { + "original": 10485768192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 0.9363505794880082, + "smooth_train_loss": 2.4582942112751494, + "total_training_time": 200133.8836593628, + "stage_start_step": 0, + "stage_training_flops": 120330745817333760000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.2033074581733376e+20 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/meta_005500.json b/experiments/Think.Unbounded-d32/base_checkpoints/meta_005500.json new file mode 100644 index 0000000000000000000000000000000000000000..2e539305014a734dedb0efd28cc56e73a826945e --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/meta_005500.json @@ -0,0 +1,236 @@ +{ + "step": 5500, + "training_complete": false, + "experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 0.8999200969809518, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "Think.Unbounded-d32", + "wandb_run_id": "9b75691b", + "wandb_group": "clean1930s-d32", + "wandb_tags": "think-dataset-clean-1930s,d32,ratio12,ctx4096,sssl,fp8,muon-momentum-constant,muon-momentum-0.90,midtrain-schedule,mix-og,0-30-60", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": 9600, + "target_flops": -1.0, + "target_param_data_ratio": -1.0, + "device_batch_size": 2, + "total_batch_size": 2097152, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/data", + "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/pretok", + "mixture_source_dirs": "{\"original\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_original\", \"midtrain_r30\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r30\", \"midtrain_r60\": \"/workspace/nanochat/experiments/Think.Unbounded-d32/pretok_midtrain_r60\"}", + "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32/base_checkpoints", + "experiment_id": "Think.Unbounded-d32", + "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "b2de514e00cde3bee362cf4c87e16d35e0f32c24", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "Think.Unbounded-d32", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" + }, + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2" + }, + "device_batch_size": 2, + "max_seq_len": 4096, + "total_batch_size": 2097152, + "dataloader_state_dict": { + "file_idx": 115, + "pos": 35752193, + "epoch": 1, + "pq_idx": 115, + "rg_idx": 35752193, + "mixture": { + "cursors": { + "original": { + "file_idx": 115, + "pos": 35752193, + "epoch": 1, + "pq_idx": 115, + "rg_idx": 35752193 + }, + "midtrain_r30": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + }, + "midtrain_r60": { + "file_idx": 0, + "pos": 0, + "epoch": 1, + "pq_idx": 0, + "rg_idx": 0 + } + }, + "cumulative_tokens": 11534344192, + "source_tokens": { + "original": 11534344192, + "midtrain_r30": 0, + "midtrain_r60": 0 + }, + "active_stage_idx": 0 + } + }, + "loop_state": { + "min_val_bpb": 0.8999200969809518, + "smooth_train_loss": 2.4195481055384045, + "total_training_time": 220148.93892598152, + "stage_start_step": 0, + "stage_training_flops": 132363820399067136000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.3236382039906714e+20 + } +} \ No newline at end of file diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_000500.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..6884c1da4747df1ea45e3a188c830568c8218441 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adc847ba9a4b6768498bbf6b84212825ac86098a137a340fd2eaefa72c01a021 +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_001000.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6edd271c451f5fe040f5cfda073887ec81dacf16 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ec635fc9fd654db070738b9ebc8b84f8b1169d60682241c8836cde1b5bf8a71 +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_001500.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..07f84dcfd96105cd888bf167f91d56ebd9f7b0d1 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2da68abcf5dbd78f3af6b00b815a538bda1955d7977ba49697b0aec3843bab2f +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_002000.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4afc57a233c2c1ad2aa774b1be488ea4978f19a7 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7e811bb7d332aa76a8593587752f27efcea20af3a03a221d2ae3a7abad4aef2 +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_002500.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_002500.pt new file mode 100644 index 0000000000000000000000000000000000000000..e87141a787975437d3ebfdbf04972e8d1492e458 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_002500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9bc3c0f1ed3f5510185c16226614281cc7187b71ddcf53447054985be764722 +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_003000.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_003000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b1a13fc0d996c94bbadf6efbdbb7eb3d3700d61 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_003000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6f2485fd307a258eb3fded1bc9b1be934ac2184fbf2eefdf10d850bba36cb13 +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_003500.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_003500.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdd15528b38ba1332fb8ca09e5da6525423ea247 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_003500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc4d43cb19a8d1c085f0d3a9ee21b023ea1ee5a4f573896f628f2f169503aeaa +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_004000.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_004000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6cea76a8152563913ca242d8c2c6a6606f2f2401 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_004000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1c498d9afbfe6ef3231f42221c888f3cf89e17ed6526b59562b40b1df47475a +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_004500.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_004500.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cc16fe8f66c528b12b60c607f1985a0cad2cc63 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_004500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bba2104bb0b0dc658dc52e9cc4c31ec0356e0d051f30e8ddb3d1dce564008ce +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_005000.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_005000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c45be74bd3c00d3a7a8c840d1822607f3d767825 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_005000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7603f520faeabb51df975a35ab8b0065e5ec14782eceb8077dca635045d05f1a +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/model_005500.pt b/experiments/Think.Unbounded-d32/base_checkpoints/model_005500.pt new file mode 100644 index 0000000000000000000000000000000000000000..59d5d22a28f4b6c92f97d4294863afe521af77a3 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/model_005500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e40a97ef3206dbb1c16ebcb1d587055eaee6438f057d150ba2f0df78345b372a +size 8992694653 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_000500_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..23ee1b6e135c15acdb5fc9bd73271bfadd44d8fe --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:060bd64c5f41a991f7822484d4856efc90683c9530eed9fb3ffbdf286428415f +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_001000_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..925866ef2023e9cd594d3ce1896f2cba6dedb74d --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b93c1121cdb0fb1a70add6e4ce189ea7d2fc79ec6b0348ecddcc5b3e8e141ac4 +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_001500_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdde4199fd7933bce9164408cfbc97c3635b275e --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18141073e073d931961e0c105037de5db7e46113e84f0bfd9faac16e92a8dd6e +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_002000_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..df33ef462ed8be0f02fc0a063a55a6333dc55fbd --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb065a47e6c313adf94f475ef87686934efc532c0e0708aa19ab1fb18778074d +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_002500_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_002500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..7938fc4fcfa6127a18b84d327c9bea9a7c299955 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_002500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b23c21030cd25dc859e938236993159aa0c951398c99904083e984cd8c9208f5 +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_003000_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_003000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1654df1b6cf3d91ad0dacefcb30ad11790aa4907 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_003000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e56c1e1a3a6e4eff55d3df7eb840e1a2ed60b9e314c1a854ccdbcf2f2a498aee +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_003500_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_003500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..81898c64322dc9dba9588808061c557fdd365637 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_003500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09542bfde4246e07a20f2f250562204f2b03c8044963b00cdb30808c357e8f94 +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_004000_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_004000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..50722b5b393b9445ebe90e25204add3f2673ddee --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_004000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0880f6566493f92ee6c594d3c59eb79c839931565a44014b8cde39340196901 +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_004500_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_004500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..729b852acebc026e69c0bd66b96df39230026402 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_004500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:045613067d60df6f12928ed7b71754c30712819a1892ecb1507b8d4786b4fd52 +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_005000_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_005000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeaacecf2c1b9dd5a333ba4f64a42e83f980cd1a --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_005000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bd4ccad84654c8e38232615469d44d622fff0348893ce28a7613db4a67845f6 +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/base_checkpoints/optim_005500_rank0.pt b/experiments/Think.Unbounded-d32/base_checkpoints/optim_005500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..603391a7f5229a41de7b825493fb3dc2e8f95054 --- /dev/null +++ b/experiments/Think.Unbounded-d32/base_checkpoints/optim_005500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fcf19df437fd1aa1f97c344b072ff93f6254e84a4733cd6af25969385faeb39 +size 11545903817 diff --git a/experiments/Think.Unbounded-d32/config.json b/experiments/Think.Unbounded-d32/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b56575dc5df745833e8b4c9d3fbc593d69ce3359 --- /dev/null +++ b/experiments/Think.Unbounded-d32/config.json @@ -0,0 +1,111 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "Think.Unbounded-d32", + "datasets": { + "original": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "45225d95bc15f942be3b4b344738cea1f66e3de8", + "validation_shard": 472, + "num_train_shards": 330, + "download_workers": 4 + }, + "midtrain_r30": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_30/data", + "validation_shard": 1262, + "num_train_shards": 391, + "download_workers": 4 + }, + "midtrain_r60": { + "adapter": "parquet_shards", + "repo": "zachnorton03/think-midtrain", + "revision": "9ace24b8e16e57e38a1ea0b1f6d7cbf323bf9dbc", + "subfolder": "mixed/ratio_60/data", + "validation_shard": 523, + "num_train_shards": 207, + "download_workers": 4 + } + }, + "mixture_schedule": { + "total_tokens": 20132659200, + "seed_data": 1234, + "max_epochs": 4, + "stages": [ + { + "name": "base", + "start_tokens": 0, + "source": "original" + }, + { + "name": "injection", + "start_tokens": 14092861440, + "source": "midtrain_r30" + }, + { + "name": "decay_mix", + "start_tokens": 18119393280, + "source": "midtrain_r60" + } + ] + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 32, + "scaling_params": 1677724672, + "target_param_data_ratio": 12, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 2, + "total_batch_size": 2097152, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "Think.Unbounded-d32", + "group": "clean1930s-d32", + "tags": [ + "think-dataset-clean-1930s", + "d32", + "ratio12", + "ctx4096", + "sssl", + "fp8", + "muon-momentum-constant", + "muon-momentum-0.90", + "midtrain-schedule", + "mix-og", + "0-30-60" + ] + }, + "config_fingerprint": "6f8ae461e0eb9ee2", + "artifact_path": "experiments/Think.Unbounded-d32" +} diff --git a/experiments/Think.Unbounded-d32/run.json b/experiments/Think.Unbounded-d32/run.json new file mode 100644 index 0000000000000000000000000000000000000000..6826cd812486d84f4a4dc7e69abc8bd1e2194015 --- /dev/null +++ b/experiments/Think.Unbounded-d32/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "Think.Unbounded-d32", + "stage": "base", + "base_experiment_id": "Think.Unbounded-d32", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "branch_parent_step": null, + "config_fingerprint": "6f8ae461e0eb9ee2", + "wandb_run_id": "9b75691b", + "created_at": 1785990696 +} diff --git a/experiments/Think.Unbounded-d32/tokenizer/experiment_tokenizer.json b/experiments/Think.Unbounded-d32/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78 --- /dev/null +++ b/experiments/Think.Unbounded-d32/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 200, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 1000000000000, + "doc_cap": 1000000000, + "vocab_size": 32768 + }, + "created_at": 1784129374 +} diff --git a/experiments/Think.Unbounded-d32/tokenizer/token_bytes.pt b/experiments/Think.Unbounded-d32/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a --- /dev/null +++ b/experiments/Think.Unbounded-d32/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2 +size 132649 diff --git a/experiments/Think.Unbounded-d32/tokenizer/tokenizer.pkl b/experiments/Think.Unbounded-d32/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac --- /dev/null +++ b/experiments/Think.Unbounded-d32/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f +size 410542 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..e41094835a7e09f53fd78b890bad5c730d50d873 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_000500.json @@ -0,0 +1,147 @@ +{ + "step": 500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "val_bpb": 1.341851696959099, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "wandb_run_id": "73b2700b", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,full-attention,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "full-attention", + "fp8" + ] + }, + "config_fingerprint": "9d3bb595965c289e", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "9d3bb595965c289e" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.341851696959099, + "smooth_train_loss": 3.7025805680376496, + "total_training_time": 571.568044424057, + "stage_training_flops": 291921016651776000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 291921016651776000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..7e937e5bd51f62ee21bb6d658eb891b1b7b6f686 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_001000.json @@ -0,0 +1,147 @@ +{ + "step": 1000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "val_bpb": 1.2316896297206081, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "wandb_run_id": "73b2700b", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,full-attention,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "full-attention", + "fp8" + ] + }, + "config_fingerprint": "9d3bb595965c289e", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "9d3bb595965c289e" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.2316896297206081, + "smooth_train_loss": 3.515503448119396, + "total_training_time": 1155.6556208133698, + "stage_training_flops": 583842033303552000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 583842033303552000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..bb1f330e9d30d34481340516d19a96df77a19914 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_001500.json @@ -0,0 +1,147 @@ +{ + "step": 1500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "val_bpb": 1.14883061633963, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "wandb_run_id": "73b2700b", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,full-attention,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "full-attention", + "fp8" + ] + }, + "config_fingerprint": "9d3bb595965c289e", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "9d3bb595965c289e" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.14883061633963, + "smooth_train_loss": 3.3998494498727863, + "total_training_time": 1739.6899712085724, + "stage_training_flops": 875763049955328000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 875763049955328000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..246080de6b5fc40cb0e09a8a6819f263a332b20c --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_002000.json @@ -0,0 +1,147 @@ +{ + "step": 2000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "val_bpb": 1.1118656088331524, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "wandb_run_id": "73b2700b", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,full-attention,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "full-attention", + "fp8" + ] + }, + "config_fingerprint": "9d3bb595965c289e", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "9d3bb595965c289e" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.1118656088331524, + "smooth_train_loss": 3.330046126752348, + "total_training_time": 2324.030104160309, + "stage_training_flops": 1167684066607104000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1167684066607104000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_002362.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..735e8ebd7601898f6f317a1e2858d9149903cb7a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/meta_002362.json @@ -0,0 +1,147 @@ +{ + "step": 2362, + "training_complete": true, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "val_bpb": 1.089810417727749, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "wandb_run_id": "73b2700b", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,full-attention,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "full-attention", + "fp8" + ] + }, + "config_fingerprint": "9d3bb595965c289e", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "9d3bb595965c289e" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38438817, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38438817 + }, + "loop_state": { + "min_val_bpb": 1.089810417727749, + "smooth_train_loss": 3.1332932870678034, + "total_training_time": 2747.30322766304, + "stage_training_flops": 1379034882662989824, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1379034882662989824 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcb4351b6ec1cf82ae77bd2537c2852d2e7ef264 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df0ad4d77d44ef41b61e65b9f2a2ef7ee6440fab7d14811fee9a3fcdc9e64e8e +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..8380f15ca9ab12b6d447e5a45824c9a482b4c33e --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2fae8d3f799e0de7533c4f89f72c8e706d7a4bce41f4b3f78b189687e532459 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..583ef04f8f412c28e802f5c7cc7125cc148f2567 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53e1ecaed8685b05ba89e76679083569bc64a982e466051e1e41dfce8cb5dd63 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b15ab3979e242fba3c1561ebabc9d264fc4e52d --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cb70f5a7eced55f0298fab3a221652724d9292db9af53ab6ce2378f0a688439 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_002362.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..b969bf1f0dac779213d4e932ceb71e69f7bfb4b5 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ecc02a89c79321d39c942b4849b2e4d8a2e7c44b74d478a9b7457cb32a8eafa +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b11bcb7eb1c34fb82fbe5478c88f1180ea6a02a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b261a4019a5d0f54a3b8b4a1f0677caeb489870e7e22107d7475a434f94c2ee0 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9779371dd1e58446a431fa5d14169f7abd86129 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83c213a1fb6ee27e243741c0337ffe1c60a659e93e44a8dd7307c7636a16dbec +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f6ced2ad593c659ac6768e623d7e9486b524ddc --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:086038b66aa5fdc1a48f5c9b366bd358080a398470e102309788ea90fd802007 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6917266ee8490b292e3147d505da50b4b67697d --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc30203e67e01d0ec2004e07c4f4aa40bfeb1e856ede30c4f126f2226f626183 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_002362_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a848cae8988f684d21b189ba3722152d1e08209 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:478bde5b6ef68865a5b698f3d24ff716442b8a350e42154a5318c57821fbdd55 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/config.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..805b44d5efcb8dd702550036c5cb2d284a1f8c3e --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/config.json @@ -0,0 +1,64 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "full-attention", + "fp8" + ] + }, + "config_fingerprint": "9d3bb595965c289e", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1" +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/core.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/core.json new file mode 100644 index 0000000000000000000000000000000000000000..f336098fd1c151a3cb49d45cdb2ff15ad51923a6 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/core.json @@ -0,0 +1,56 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": 0.07475136478377907, + "core_results": { + "hellaswag_zeroshot": 0.2768372893333435, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.08469071239233017, + "arc_easy": 0.3308080732822418, + "arc_challenge": 0.22440272569656372, + "copa": 0.5099999904632568, + "commonsense_qa": 0.315315306186676, + "piqa": 0.5511425137519836, + "openbook_qa": 0.2160000056028366, + "lambada_openai": 0.21793130040168762, + "hellaswag": 0.2775343358516693, + "winograd": 0.6043956279754639, + "winogrande": 0.5011838674545288, + "bigbench_dyck_languages": 0.1080000028014183, + "agi_eval_lsat_ar": 0.25652173161506653, + "bigbench_cs_algorithms": 0.40303027629852295, + "bigbench_operators": 0.05714286118745804, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.01106906309723854, + "coqa": 0.06363522261381149, + "boolq": 0.5709480047225952, + "bigbench_language_identification": 0.25380000472068787 + }, + "centered_results": { + "hellaswag_zeroshot": 0.03578305244445801, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.08469071239233017, + "arc_easy": 0.10774409770965576, + "arc_challenge": -0.034129699071248375, + "copa": 0.019999980926513672, + "commonsense_qa": 0.144144132733345, + "piqa": 0.10228502750396729, + "openbook_qa": -0.04533332586288452, + "lambada_openai": 0.21793130040168762, + "hellaswag": 0.036712447802225746, + "winograd": 0.20879125595092773, + "winogrande": 0.002367734909057617, + "bigbench_dyck_languages": 0.1080000028014183, + "agi_eval_lsat_ar": 0.07065216451883315, + "bigbench_cs_algorithms": 0.40303027629852295, + "bigbench_operators": 0.05714286118745804, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.01106906309723854, + "coqa": 0.06363522261381149, + "boolq": -0.12908419809843363, + "bigbench_language_identification": 0.17909791498425506 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/samples.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..f0c2ae157ca859bc54b233e46e77aa27b47d9cef --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the kingdom of France. The capital of the kingdom of France is" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the same as that of the sun, and the same as that of the sun" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. I have been in the habit of going to the house of Mr." + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as the opposite of cold. The former is the same as the latter" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a dark, dark, and dark color, and I have no doubt that the" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the parts of the body which are in the body, and the" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay was contributed to the \"Introduction to English Literature,\" by the celebrated Mr. A. H.\n\nSmithfield Windle, the author of the Life of Windle, LL.D.\n\nTerritorial Entry Book in Faneuil Hall, Norwich.\n\nBeing a compilation from the Journals of the company which Dunning called the \"Baldive,\" or Committee of the House of Commons, held at a time when the woollen manufacture of New England was taking its full advantage to wear more favourable conditions, it is now very little altered from what it really is.\n\nIn order to have the best assortment of text-books for", + "<|bos|> members were privileged, not asinders, but as guests. Various expressions, for which privileges and responsions would be legal, were freely, but imperfectly disposed of at the time of their introduction, and, so far as it was possible, nothing could serve them except the courtesy of those friendly, courteous, and communicative powers. These statements, we are glad to infer, were understood and appro- 172. priated by friendship. Some leaders of great opinion, such as Joseph Brawns, Van\n\nJoseph reviewers, and all the other leaders of the present war, confined their influence to the side of excluding those who", + "<|bos|>The vision of the man who is mighty to convert brute animals-a man also worthy of the unity which it reveals, wherein he leads a world that is new in its way-Military civilization has attained its unity. Story\n\nApril 10. The vision first revealed to the cison, William\n\nFemale Arnoy as a young warrior who had to find a place in another and more attractive world-a man who formed the purpose of the world of the future-he became first and chiefly interested in the progress of moral and spiritual emancipation which has since made it the aim and object of that great impulse in advance. Essay on the Form of", + "<|bos|>ESLIE'S CHIEFRY.\n\nXX\n\nION.\n\n12 CHANGE DEMOX.\n\nEVE\n\nFAGE OF OPENING THE NILE.\n\nN RATTENOORKS\n\nJEREMIAH PARKER.\n\nNABIKIE BONDLEY.\n\nRAPP SOUTH RADER.\n\nTHE ROCK DAY.\n\nDYCE HIS HOUSE ON A WELL.\n\nPRICES.\n\nPALSTATE OF A CONVERT HOME.\n\nREOTHING\n\nTRANSFORT.\n\nSUPERSET SOUTH AMERICA.\n\n[LIST.]\n\nStraight SPEECH OF CHARLES A. Thacher.\n\nE DELHAND.-THE ROOTS - MEDALS", + "<|bos|>CHAPTER II.\n\nTHE TIDE OF MOUNTAIN PUNISHMENT.\n\nTHE gold has risen by slow stages, but averaged by a slow process. Then it was that the sea, with its red currents running through it, rushed in and pursued its great voyage; and the whole country in full wonder at our eagerness soon seemed to vanish, of the torrents of blood rushing in, on either side, like liquid salt streams [430] ready to overflow. And at last this mixture of wind and tide came into use [441] much as in the ancient days of that ancient city the sea called Adria\n\nCONCLUDED", + "<|bos|>PREFACE\n\nand PERS. likewise. But pictures and pictures of the passions pass into the hands of the beholder after they are painted: at first, but not before, it is the furthest stage of thoughts-catching always that pursuit the quiet enjoyment, and the most immediate order, of nature.\n\nA people so polished that they cannot look at people without perceiving much vanity and often whimsicality, an expression of fine sense, a thousand ways excelling the common language of the middle ages, and that modes of thought are the most subtle, rare, and beautiful that could have been devised.\n\nAnd, lastly, the feelings when painted are", + "<|bos|>Clatim. Sultan\n\nLocker v. Manchester, &c., Co., 34 Fed. 222.\n\n125a. \u00fc 180 a; 3.5 P.\n\n180 a; 7.8 P. b. 181 a; 8.79 P. b. 182; 9.2 P. b. 185a; 10.4 P. b. 187q.\n\n184 a; 11.6 P. b. 187 b; 12.5 P. b. 185 b.\n\n18", + "<|bos|>Lestr Menderella and Victoria.\" \"You know that that parent of ours the conductor was not forgotten in this place.\" \"For many days large numbers of People were detained at Passes.\" \"About four people intervened, chiefly middle-sized disreputable boys, unaccus-\n\ntimed with that kind of respectability which Blot had never reached but in those days when life was so unsatisfactory.\" \"Do you remember having acted as conductor?\" \"I remember easily enough.\" \"Well, I know about very many people.\" \"6 \"Ah, is it true that Representatives are allowed to become securities for themselves?\"\n\nMass. \"Disability" + ] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/val_bpb.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..9a599e62a64f4de5ba63996a92ad41450547a366 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/evals/val_bpb.json @@ -0,0 +1,12 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val": 1.0353479590828456 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/run.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/run.json new file mode 100644 index 0000000000000000000000000000000000000000..3b2d8ae0bfa1355de190d5b667536837e8d89fc4 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "9d3bb595965c289e", + "wandb_run_id": "73b2700b", + "created_at": 1784316816 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/meta_000021.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/meta_000021.json new file mode 100644 index 0000000000000000000000000000000000000000..8415c6a50964d1b2b6e8abe793895dcf379170d2 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/meta_000021.json @@ -0,0 +1,114 @@ +{ + "step": 21, + "training_complete": true, + "val_bpb": 0.9112977907020111, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5", + "wandb_run_id": "332d9c75", + "wandb_group": "think-d12", + "wandb_tags": "sft,pre1930-routes,stem5", + "device_type": "", + "model_tag": null, + "model_step": null, + "base_checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/base_checkpoints", + "base_step": 2362, + "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints", + "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer", + "resume_from_step": null, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5", + "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/config.json", + "parent_cumulative_flops": 1.3790348826629898e+18, + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "fd70d9abb7eaeffa712c8b9665d7d9e7495e9ee8", + "load_optimizer": 1, + "num_iterations": -1, + "max_seq_len": null, + "device_batch_size": 8, + "total_batch_size": null, + "embedding_lr": null, + "unembedding_lr": null, + "matrix_lr": null, + "init_lr_frac": 0.8, + "warmup_ratio": 0.0, + "warmdown_ratio": 0.5, + "final_lr_frac": 0.0, + "eval_every": -1, + "eval_tokens": 20971520, + "chatcore_every": -1, + "chatcore_max_cat": -1, + "chatcore_max_sample": 24, + "save_every": -1, + "recipe": "pre1930-routes", + "pre1930_epochs": 5, + "mmlu_epochs": 3, + "gsm8k_epochs": 4, + "authentic_epochs": 0, + "knowledge_qa_epochs": 0, + "multiturn_qa_epochs": 0, + "reasoning_qa_epochs": 0, + "stem_reasoning_epochs": 0, + "narrative_grounded_epochs": 0, + "narrative_fiction_epochs": 0, + "opinion_qa_epochs": 0, + "how_to_qa_epochs": 0, + "verse_qa_epochs": 0, + "composition_qa_epochs": 0, + "resolved_experiment_config": { + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-synth-stem5", + "data": { + "recipe": "pre1930-routes", + "stem_reasoning_epochs": 5 + }, + "training": { + "num_iterations": -1, + "device_batch_size": 8, + "eval_every": -1, + "chatcore_every": -1, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d12", + "tags": [ + "sft", + "pre1930-routes", + "stem5" + ] + }, + "config_fingerprint": "0e7e06cac4e1dfb1", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5" + }, + "stage": "sft", + "base_experiment_id": null, + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "0e7e06cac4e1dfb1" + }, + "loop_state": { + "step": 21, + "total_training_time": 33.068419218063354, + "min_val_bpb": 0.9112977907020111, + "smooth_train_loss": 1.645360864787687, + "mfu": 62.1129491496942, + "tok_per_sec": 174024, + "stage_training_flops": 1.2260682699374592e+16, + "inherited_parent_flops": 1.3790348826629898e+18, + "cumulative_pipeline_training_flops": 1.3912955653623644e+18 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/model_000021.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/model_000021.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cf766ea5946859a6cdc68f642b422435875c84a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/model_000021.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fae981d730dbf371edb3d15aa2a020f9eb537832c56ce2994ad4c2146e85aac +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/optim_000021_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/optim_000021_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9f9fe7a6a459159c4e2114a4447ba0ac26ee390 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/checkpoints/optim_000021_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27e05c2f420da92d2c012237a58c1cb244a8392321e5bf99e2254cb16937b013 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/config.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/config.json new file mode 100644 index 0000000000000000000000000000000000000000..87940571e48485bc5be39f1fede643a2bb88d6b0 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/config.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-synth-stem5", + "data": { + "recipe": "pre1930-routes", + "stem_reasoning_epochs": 5 + }, + "training": { + "num_iterations": -1, + "device_batch_size": 8, + "eval_every": -1, + "chatcore_every": -1, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d12", + "tags": [ + "sft", + "pre1930-routes", + "stem5" + ] + }, + "config_fingerprint": "0e7e06cac4e1dfb1", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5" +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/run.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/run.json new file mode 100644 index 0000000000000000000000000000000000000000..08da5f6e1d17e1c9f959a644e19df4fff299f8ce --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5", + "stage": "sft", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_checkpoint_step": null, + "branch_parent_step": null, + "config_fingerprint": "0e7e06cac4e1dfb1", + "wandb_run_id": "332d9c75", + "created_at": 1785518357 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/summary.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9fd85fa3cd4b765bcb1775ef550c4c8cdfcf225b --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5/summary.json @@ -0,0 +1,19 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5", + "stage": "sft", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_checkpoint_step": null, + "step": 21, + "stage_training_flops": 1.2260682699374592e+16, + "inherited_parent_flops": 1.3790348826629898e+18, + "cumulative_pipeline_training_flops": 1.3912955653623644e+18, + "chatcore_metric": null, + "chat_results": null, + "training_time_seconds": 33.068419218063354, + "config_fingerprint": "0e7e06cac4e1dfb1", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "fd70d9abb7eaeffa712c8b9665d7d9e7495e9ee8", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/332d9c75", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/sft/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1-pre1930-synth-stem5" +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/summary.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4bc7c8744379cebbd9c9c771b88ef6f97596318b --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/summary.json @@ -0,0 +1,91 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset-clean-1930s", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.089810417727749, + "minimum_sampled_val_bpb": 1.089810417727749, + "full_val_bpb": 1.0353479590828456, + "core_metric": 0.07475136478377907, + "centered_results": { + "hellaswag_zeroshot": 0.03578305244445801, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.08469071239233017, + "arc_easy": 0.10774409770965576, + "arc_challenge": -0.034129699071248375, + "copa": 0.019999980926513672, + "commonsense_qa": 0.144144132733345, + "piqa": 0.10228502750396729, + "openbook_qa": -0.04533332586288452, + "lambada_openai": 0.21793130040168762, + "hellaswag": 0.036712447802225746, + "winograd": 0.20879125595092773, + "winogrande": 0.002367734909057617, + "bigbench_dyck_languages": 0.1080000028014183, + "agi_eval_lsat_ar": 0.07065216451883315, + "bigbench_cs_algorithms": 0.40303027629852295, + "bigbench_operators": 0.05714286118745804, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.01106906309723854, + "coqa": 0.06363522261381149, + "boolq": -0.12908419809843363, + "bigbench_language_identification": 0.17909791498425506 + }, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the kingdom of France. The capital of the kingdom of France is" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the same as that of the sun, and the same as that of the sun" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. I have been in the habit of going to the house of Mr." + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as the opposite of cold. The former is the same as the latter" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a dark, dark, and dark color, and I have no doubt that the" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the parts of the body which are in the body, and the" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay was contributed to the \"Introduction to English Literature,\" by the celebrated Mr. A. H.\n\nSmithfield Windle, the author of the Life of Windle, LL.D.\n\nTerritorial Entry Book in Faneuil Hall, Norwich.\n\nBeing a compilation from the Journals of the company which Dunning called the \"Baldive,\" or Committee of the House of Commons, held at a time when the woollen manufacture of New England was taking its full advantage to wear more favourable conditions, it is now very little altered from what it really is.\n\nIn order to have the best assortment of text-books for", + "<|bos|> members were privileged, not asinders, but as guests. Various expressions, for which privileges and responsions would be legal, were freely, but imperfectly disposed of at the time of their introduction, and, so far as it was possible, nothing could serve them except the courtesy of those friendly, courteous, and communicative powers. These statements, we are glad to infer, were understood and appro- 172. priated by friendship. Some leaders of great opinion, such as Joseph Brawns, Van\n\nJoseph reviewers, and all the other leaders of the present war, confined their influence to the side of excluding those who", + "<|bos|>The vision of the man who is mighty to convert brute animals-a man also worthy of the unity which it reveals, wherein he leads a world that is new in its way-Military civilization has attained its unity. Story\n\nApril 10. The vision first revealed to the cison, William\n\nFemale Arnoy as a young warrior who had to find a place in another and more attractive world-a man who formed the purpose of the world of the future-he became first and chiefly interested in the progress of moral and spiritual emancipation which has since made it the aim and object of that great impulse in advance. Essay on the Form of", + "<|bos|>ESLIE'S CHIEFRY.\n\nXX\n\nION.\n\n12 CHANGE DEMOX.\n\nEVE\n\nFAGE OF OPENING THE NILE.\n\nN RATTENOORKS\n\nJEREMIAH PARKER.\n\nNABIKIE BONDLEY.\n\nRAPP SOUTH RADER.\n\nTHE ROCK DAY.\n\nDYCE HIS HOUSE ON A WELL.\n\nPRICES.\n\nPALSTATE OF A CONVERT HOME.\n\nREOTHING\n\nTRANSFORT.\n\nSUPERSET SOUTH AMERICA.\n\n[LIST.]\n\nStraight SPEECH OF CHARLES A. Thacher.\n\nE DELHAND.-THE ROOTS - MEDALS", + "<|bos|>CHAPTER II.\n\nTHE TIDE OF MOUNTAIN PUNISHMENT.\n\nTHE gold has risen by slow stages, but averaged by a slow process. Then it was that the sea, with its red currents running through it, rushed in and pursued its great voyage; and the whole country in full wonder at our eagerness soon seemed to vanish, of the torrents of blood rushing in, on either side, like liquid salt streams [430] ready to overflow. And at last this mixture of wind and tide came into use [441] much as in the ancient days of that ancient city the sea called Adria\n\nCONCLUDED", + "<|bos|>PREFACE\n\nand PERS. likewise. But pictures and pictures of the passions pass into the hands of the beholder after they are painted: at first, but not before, it is the furthest stage of thoughts-catching always that pursuit the quiet enjoyment, and the most immediate order, of nature.\n\nA people so polished that they cannot look at people without perceiving much vanity and often whimsicality, an expression of fine sense, a thousand ways excelling the common language of the middle ages, and that modes of thought are the most subtle, rare, and beautiful that could have been devised.\n\nAnd, lastly, the feelings when painted are", + "<|bos|>Clatim. Sultan\n\nLocker v. Manchester, &c., Co., 34 Fed. 222.\n\n125a. \u00fc 180 a; 3.5 P.\n\n180 a; 7.8 P. b. 181 a; 8.79 P. b. 182; 9.2 P. b. 185a; 10.4 P. b. 187q.\n\n184 a; 11.6 P. b. 187 b; 12.5 P. b. 185 b.\n\n18", + "<|bos|>Lestr Menderella and Victoria.\" \"You know that that parent of ours the conductor was not forgotten in this place.\" \"For many days large numbers of People were detained at Passes.\" \"About four people intervened, chiefly middle-sized disreputable boys, unaccus-\n\ntimed with that kind of respectability which Blot had never reached but in those days when life was so unsatisfactory.\" \"Do you remember having acted as conductor?\" \"I remember easily enough.\" \"Well, I know about very many people.\" \"6 \"Ah, is it true that Representatives are allowed to become securities for themselves?\"\n\nMass. \"Disability" + ], + "training_time_seconds": 2747.30322766304, + "stage_training_flops": 1.3790348826629898e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.3790348826629898e+18, + "config_fingerprint": "9d3bb595965c289e", + "git_commit_sha": "fd70d9abb7eaeffa712c8b9665d7d9e7495e9ee8", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/73b2700b", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1", + "dataset_fingerprint": "0db4605cfe3a7eac", + "tokenizer_fingerprint": "21e99d5cdeeaa660" +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 200, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 1000000000000, + "doc_cap": 1000000000, + "vocab_size": 32768 + }, + "created_at": 1784129374 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2 +size 132649 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-full-fulltok-ablation-v1/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f +size 410542 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..94b33333495b63f89661f3b35a1d9736afad22f5 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_000500.json @@ -0,0 +1,147 @@ +{ + "step": 500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "val_bpb": 1.3388062223680175, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "wandb_run_id": "1528c501", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,sssl,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "sssl", + "fp8" + ] + }, + "config_fingerprint": "b2b7a42993186da5", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b2b7a42993186da5" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3388062223680175, + "smooth_train_loss": 3.697135257265452, + "total_training_time": 529.8762292861938, + "stage_training_flops": 225125685264384000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 225125685264384000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..3928cf6a33941108da94342e3dc4aaac807f1b5b --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_001000.json @@ -0,0 +1,147 @@ +{ + "step": 1000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "val_bpb": 1.232943966369166, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "wandb_run_id": "1528c501", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,sssl,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "sssl", + "fp8" + ] + }, + "config_fingerprint": "b2b7a42993186da5", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b2b7a42993186da5" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.232943966369166, + "smooth_train_loss": 3.515408592820662, + "total_training_time": 1071.7539193630219, + "stage_training_flops": 450251370528768000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 450251370528768000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..e3f31ff300dc12b88e88a45aefad63d085b48761 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_001500.json @@ -0,0 +1,147 @@ +{ + "step": 1500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "val_bpb": 1.1509991902459196, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "wandb_run_id": "1528c501", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,sssl,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "sssl", + "fp8" + ] + }, + "config_fingerprint": "b2b7a42993186da5", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b2b7a42993186da5" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.1509991902459196, + "smooth_train_loss": 3.402263473061546, + "total_training_time": 1613.613620042801, + "stage_training_flops": 675377055793152000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 675377055793152000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..f2ed72554a14e215c6f4b0e221cc2e39ffbf8dea --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_002000.json @@ -0,0 +1,147 @@ +{ + "step": 2000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "val_bpb": 1.1138402204171254, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "wandb_run_id": "1528c501", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,sssl,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "sssl", + "fp8" + ] + }, + "config_fingerprint": "b2b7a42993186da5", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b2b7a42993186da5" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.1138402204171254, + "smooth_train_loss": 3.335568630966098, + "total_training_time": 2155.4914762973785, + "stage_training_flops": 900502741057536000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 900502741057536000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_002362.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..661ac6ff883ea5052e245450711a365af41e5720 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/meta_002362.json @@ -0,0 +1,147 @@ +{ + "step": 2362, + "training_complete": true, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "val_bpb": 1.09204857698696, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "wandb_run_id": "1528c501", + "wandb_group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,attention-ablation-v1,sssl,fp8", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/pretok", + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "sssl", + "fp8" + ] + }, + "config_fingerprint": "b2b7a42993186da5", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b2b7a42993186da5" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38438817, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38438817 + }, + "loop_state": { + "min_val_bpb": 1.09204857698696, + "smooth_train_loss": 3.137487900090049, + "total_training_time": 2547.9463839530945, + "stage_training_flops": 1063493737188950016, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1063493737188950016 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..deb1c4c403877355afe30b26c3e5c8cc49ce4f90 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fa951ce068a1afecc753d53d603601b76d79a3b0450e27ad87d80b79d030c45 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..75eac5cd3ac4d45315a06491a6d7329994d968db --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e89c40c8e8c831f2b0e9e4e152b98519c9e39f0096b65d34c3dcb18da7419b5f +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..14cc7c365def29144921f701961c7f64d7f4f20e --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04f941a4497c06fe8a7915babe96271d21742a4c7c7d81c84ef456831fec6820 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7603ff8f49ed5be0a3a5f1b9f168ff30387d3aef --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8db77b39d2492a0625d5252493816b36affe1a3b71cfe913d13e57dab33a88f5 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_002362.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e23b95850ae6d44b7abfa6e488a0e8737f701ae --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07a30cd3d55ad8c34531d38822c0e49eb80ce81acdca53ee39ef4095b098b89c +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..43968ab486eb98708ad8cc4709469213040ff872 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1241b441b7b3c4b7c6670e0448f1c643ef76b2e29aecb20bbd8e9b47b4c741c0 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e5fc814f982e311d0ab1b65741b05314c6dd2f6 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d53d042be62262610cd82685ab9cb7988349f902de2a662cbcc7bd3b6be358ad +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9e99a3abfa6064953f06e3c511b40f127fc4f04 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7e5e0fb40d9e8da1088b7eb13779d3b40c35d78d2f92c598d77a5e870401fa2 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8a951c7ca9e7c6be3b9ce4127bc15bc818ae32e --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8789a8db6987ebaf83061895c81115cd55de86023eaa2dc9f3cf43068da1abe6 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_002362_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f91834de9820ac617717768274f8ce44f4761f1 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73905482cf6275645c38ef7abb576f97b07a3b694b3d50f8f083ccbe955321bd +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/config.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..206aa1fd4c2cfb3437d45fee41c15a6cfe2c9a8c --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/config.json @@ -0,0 +1,64 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "group": "clean1930s-d12-ctx4096-attention-ablation-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "attention-ablation-v1", + "sssl", + "fp8" + ] + }, + "config_fingerprint": "b2b7a42993186da5", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1" +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/evals/samples.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..e8abb0c133117628d739cc248892daff0f95e4b2 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the kingdom of France, and the capital of the kingdom of England" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the earth. The symbol of the gold of the" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If yesterday is Saturday, then tomorrow will be Saturday. If yesterday is" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as the opposite of cold. The latter is the same as the former" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the red, and I have a great many other colors. I have a great" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of days in the year, and x is the number of days in" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay was contributed to the \"Preliminary.\" Its author says \"This work is undertaken for candidates who love the genius and industry of the manufacturing people of the United States. Among those who love the Cliff institutions in Faneuil Hall, Fort Hall, Manchester, and Price's ostentatious company, Dunning, Upton, Tecumseh, Whittaker & Co. Stockport, Stockport, Rutland, Buckingham, Cole, & Co., & Co., are included.\"* The Address to the People of Faneuil Hall, Garrisonville, is published, is by Mr. Hoikum, and", + "<|bos|>PREFACE.\n\nTHE record of the Scotch courts of justice from 1807 to 1812 is published with the assistance of notes by\n\nGeorge W. Ellis, of New South Wales. The studies prosecuted in Scottish courts during the past year were 62 Scotch organized courts in the county of Cumberland, the Causes of the\n\nCauses of the first district court court court courts court courts court courts courts court court courts court courts courts court courts court courts court court courts court court courts courts court courts court courts court courts court courts court review court courts court court court courts court courts court court court courts courts court courts court courts court courts court court court", + "<|bos|>The vision ended with an enumeration of Wilson's impressions on Dyke's memory.\n\nA little sequel then went forth, wherein I find a world of tender notes in Life's Present Space, delivered upon the Religious Assemblics, now greatly refreshed and strengthened. These are copied c. V.\n\nINTRODUCTION OF THE ATHELLO'S FRIENDShip.\n\n105 upon the virtues' and what they are, which but formed the background of the diverse portraits conceived for each age. Long ago that painter died, like Saul and\n\nJonathan, among theocrats. On his death cancel that great travail in England. Certes, the", + "<|bos|> ministered to 350,752 zeal for the one great institution of learning, or Christian faith, which Christ died to be remembered by his disciples. William Rutherford, in 1840, was a farmer; and the \"Missionary Bldg. and Farmer Steenin, in their investigations as to the rate of salary allowed in England 50 years ago, have shown, that charges of this kind (which consist chiefly of percentages) are too high. Even 3 years last past beer-and-day rate would consequently be but little in vogue this season.\n\nWere we to have another Sunday-school at -2-y", + "<|bos|>eed\n\nNew York paid them her separate yearly pension of \u00a33,700 a year. It would be quite sufficient to hustle the\n\nCrown publicly against the attacks of such visitants. But to imprison her was still not allowed; she would break down. The subsidy in her stead was at our disposal. In April, 1916, certain British subjects started to attack America, with the view of bringing about a peace with Madagascar. And this, on their part, was successfully accomplished.\n\nDivers wooden beams were shot across the country. Since 1913 the preparation of A. H. Wright's paper for", + "<|bos|>PREFACE\n\n771. The property when pictures and pictures was being discussed at the hotel, and party papers, letters, etiquette, the French management, and of course the general reception of the day; and thoughts of it always took place in household affairs, and on most public questions, and by fair and regular means it was not unfrequently found to match Fish's bill book and \"possibly \" an opportunity of getting expressions of reform from her chairman, and among many other ceremonials in her favour, and Christian Orientalizing the German people, and breaking the two latter semi-faces.\n\nIt sometimes happened, as the feelings of the audience", + "<|bos|>) have they not repeated in the Bible the same injunction, \"join unto all the rest of the world.\" What do these seven epistles contain? It 1\n\nAnd do metaphorical language not admit of a similar application? smell no of blood\" it is simply and solely by thinking they are convenient.\n\nBut are you thinking about blood? Are your expenses great? Is there nothing but blood which you use?\n\nDo you read the book of which I write ?\"\n\nIs there nothing but rosewood? Is your house full for gardening? Is there fine linen for bloomery? If so, how much more for open doors? But how", + "<|bos|>PREFACE.\n\nbut Mendez considers himself rewarded in a large domain by that freedom of epistolary intercourse from which he had so long endured a painful bondage.\n\nHe felt that neither the virtue of his literary embrasure nor the man would bear disreputation so long as he lived.\n\nThat he should have seized upon that which constituted the most consoling but the direst part of his destiny may be doubted; though he well knew that Mendez looked forward with an agonizing pang to the end. But he did not confine himself to the object of discipline; he aspired to a familiar intercourse with his native land. And yet perhaps even" + ] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/evals/val_bpb.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..b8f3d9844a0075cff3bb0d0487b33db49732e8c5 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/evals/val_bpb.json @@ -0,0 +1,12 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val": 1.0373025644430633 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/run.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/run.json new file mode 100644 index 0000000000000000000000000000000000000000..6917f8ebff63c2d8d95dbdbe097b22767b686453 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b2b7a42993186da5", + "wandb_run_id": "1528c501", + "created_at": 1784313993 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/summary.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89d86def6317fefba6d18b3d9c8bdb40c6de5b --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/summary.json @@ -0,0 +1,69 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset-clean-1930s", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.09204857698696, + "minimum_sampled_val_bpb": 1.09204857698696, + "full_val_bpb": 1.0373025644430633, + "core_metric": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the kingdom of France, and the capital of the kingdom of England" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the earth. The symbol of the gold of the" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If yesterday is Saturday, then tomorrow will be Saturday. If yesterday is" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as the opposite of cold. The latter is the same as the former" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the red, and I have a great many other colors. I have a great" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of days in the year, and x is the number of days in" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay was contributed to the \"Preliminary.\" Its author says \"This work is undertaken for candidates who love the genius and industry of the manufacturing people of the United States. Among those who love the Cliff institutions in Faneuil Hall, Fort Hall, Manchester, and Price's ostentatious company, Dunning, Upton, Tecumseh, Whittaker & Co. Stockport, Stockport, Rutland, Buckingham, Cole, & Co., & Co., are included.\"* The Address to the People of Faneuil Hall, Garrisonville, is published, is by Mr. Hoikum, and", + "<|bos|>PREFACE.\n\nTHE record of the Scotch courts of justice from 1807 to 1812 is published with the assistance of notes by\n\nGeorge W. Ellis, of New South Wales. The studies prosecuted in Scottish courts during the past year were 62 Scotch organized courts in the county of Cumberland, the Causes of the\n\nCauses of the first district court court court courts court courts court courts courts court court courts court courts courts court courts court courts court court courts court court courts courts court courts court courts court courts court courts court review court courts court court court courts court courts court court court courts courts court courts court courts court courts court court court", + "<|bos|>The vision ended with an enumeration of Wilson's impressions on Dyke's memory.\n\nA little sequel then went forth, wherein I find a world of tender notes in Life's Present Space, delivered upon the Religious Assemblics, now greatly refreshed and strengthened. These are copied c. V.\n\nINTRODUCTION OF THE ATHELLO'S FRIENDShip.\n\n105 upon the virtues' and what they are, which but formed the background of the diverse portraits conceived for each age. Long ago that painter died, like Saul and\n\nJonathan, among theocrats. On his death cancel that great travail in England. Certes, the", + "<|bos|> ministered to 350,752 zeal for the one great institution of learning, or Christian faith, which Christ died to be remembered by his disciples. William Rutherford, in 1840, was a farmer; and the \"Missionary Bldg. and Farmer Steenin, in their investigations as to the rate of salary allowed in England 50 years ago, have shown, that charges of this kind (which consist chiefly of percentages) are too high. Even 3 years last past beer-and-day rate would consequently be but little in vogue this season.\n\nWere we to have another Sunday-school at -2-y", + "<|bos|>eed\n\nNew York paid them her separate yearly pension of \u00a33,700 a year. It would be quite sufficient to hustle the\n\nCrown publicly against the attacks of such visitants. But to imprison her was still not allowed; she would break down. The subsidy in her stead was at our disposal. In April, 1916, certain British subjects started to attack America, with the view of bringing about a peace with Madagascar. And this, on their part, was successfully accomplished.\n\nDivers wooden beams were shot across the country. Since 1913 the preparation of A. H. Wright's paper for", + "<|bos|>PREFACE\n\n771. The property when pictures and pictures was being discussed at the hotel, and party papers, letters, etiquette, the French management, and of course the general reception of the day; and thoughts of it always took place in household affairs, and on most public questions, and by fair and regular means it was not unfrequently found to match Fish's bill book and \"possibly \" an opportunity of getting expressions of reform from her chairman, and among many other ceremonials in her favour, and Christian Orientalizing the German people, and breaking the two latter semi-faces.\n\nIt sometimes happened, as the feelings of the audience", + "<|bos|>) have they not repeated in the Bible the same injunction, \"join unto all the rest of the world.\" What do these seven epistles contain? It 1\n\nAnd do metaphorical language not admit of a similar application? smell no of blood\" it is simply and solely by thinking they are convenient.\n\nBut are you thinking about blood? Are your expenses great? Is there nothing but blood which you use?\n\nDo you read the book of which I write ?\"\n\nIs there nothing but rosewood? Is your house full for gardening? Is there fine linen for bloomery? If so, how much more for open doors? But how", + "<|bos|>PREFACE.\n\nbut Mendez considers himself rewarded in a large domain by that freedom of epistolary intercourse from which he had so long endured a painful bondage.\n\nHe felt that neither the virtue of his literary embrasure nor the man would bear disreputation so long as he lived.\n\nThat he should have seized upon that which constituted the most consoling but the direst part of his destiny may be doubted; though he well knew that Mendez looked forward with an agonizing pang to the end. But he did not confine himself to the object of discipline; he aspired to a familiar intercourse with his native land. And yet perhaps even" + ], + "training_time_seconds": 2547.9463839530945, + "stage_training_flops": 1.06349373718895e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.06349373718895e+18, + "config_fingerprint": "b2b7a42993186da5", + "git_commit_sha": "c850ee9513ff3af301af2b7ff127e8709a4f6d49", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/1528c501", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1", + "dataset_fingerprint": "0db4605cfe3a7eac", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "unique_train_tokens": 0 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 200, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 1000000000000, + "doc_cap": 1000000000, + "vocab_size": 32768 + }, + "created_at": 1784129374 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2 +size 132649 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-ablation-v1/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f +size 410542 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..88494617d3b7d1c0073098f4957218d77197ae2d --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_000500.json @@ -0,0 +1,161 @@ +{ + "step": 500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.340710622409166, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "wandb_run_id": "dd182242", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,current-code-baseline,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": null, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "current-code-baseline", + "seed42" + ] + }, + "config_fingerprint": "b389d34fe603d8f3", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b389d34fe603d8f3" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.340710622409166, + "smooth_train_loss": 3.699678794303341, + "total_training_time": 527.217556476593, + "stage_start_step": 0, + "stage_training_flops": 225125685264384000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2.25125685264384e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..c73804a90519745bce827188e19f968415b2875a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_001000.json @@ -0,0 +1,161 @@ +{ + "step": 1000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.2318290352476646, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "wandb_run_id": "dd182242", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,current-code-baseline,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": null, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "current-code-baseline", + "seed42" + ] + }, + "config_fingerprint": "b389d34fe603d8f3", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b389d34fe603d8f3" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.2318290352476646, + "smooth_train_loss": 3.516440409823289, + "total_training_time": 1065.5147593021393, + "stage_start_step": 0, + "stage_training_flops": 450251370528768000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 4.50251370528768e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..964eb90ce91be860ae7a0f3bdc7df5ee03276151 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_001500.json @@ -0,0 +1,161 @@ +{ + "step": 1500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.149810760084606, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "wandb_run_id": "dd182242", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,current-code-baseline,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": null, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "current-code-baseline", + "seed42" + ] + }, + "config_fingerprint": "b389d34fe603d8f3", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b389d34fe603d8f3" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.149810760084606, + "smooth_train_loss": 3.402021092193979, + "total_training_time": 1603.780502319336, + "stage_start_step": 0, + "stage_training_flops": 675377055793152000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 6.75377055793152e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..a62f0550339c9d055185db54450c207bf3ade26e --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_002000.json @@ -0,0 +1,161 @@ +{ + "step": 2000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.1138188516867007, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "wandb_run_id": "dd182242", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,current-code-baseline,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": null, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "current-code-baseline", + "seed42" + ] + }, + "config_fingerprint": "b389d34fe603d8f3", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b389d34fe603d8f3" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.1138188516867007, + "smooth_train_loss": 3.3352839560698695, + "total_training_time": 2142.2500989437103, + "stage_start_step": 0, + "stage_training_flops": 900502741057536000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 9.00502741057536e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_002362.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..2436b6899e9a97733035345eab9f884e80c402bf --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/meta_002362.json @@ -0,0 +1,161 @@ +{ + "step": 2362, + "training_complete": true, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.0917705486702252, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "wandb_run_id": "dd182242", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,current-code-baseline,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": null, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "current-code-baseline", + "seed42" + ] + }, + "config_fingerprint": "b389d34fe603d8f3", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "b389d34fe603d8f3" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38438817, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38438817 + }, + "loop_state": { + "min_val_bpb": 1.0917705486702252, + "smooth_train_loss": 3.138216938328097, + "total_training_time": 2532.0307886600494, + "stage_start_step": 0, + "stage_training_flops": 1063493737188950016, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.06349373718895e+18 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8613958f0070f20b4302b01ce2a73bd504d24db --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5618c95938c3ffaed59cafc1817f4785748f0993dc56080c8a1adf980bbf3f +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a13e6ad641fe851d006e330d14faacf8f7710c97 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfd8c9fa9b4cdcc67b3b1f21100bcac0e24b245573070b81a2e2f46e5a1ca288 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..e313b2c3420c1eb1b8dd69a85420600d354ea10e --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7efa0c331acb4544d2fbf7d3cb43df147912bd75d5ab795b537395881780bf07 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb4bda0c0db5971b0b660974d24255daca75e776 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d36859bf90675e49623612243c416b178d76a948ee3c430d4924f33c26897b +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_002362.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..380f0e1548f98a6a87d93b6cb61722bd9f122290 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a51d4b72445597a7798b15fa5d7133a16a69f7787cda73fb57f2776adda55cef +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..efd7d94dc795b207e0f6dcae53077ed27c712175 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1328c5fac82a4f04f8d848a8fa242cfaa574aab99cdfe7f41b70d89ff73abdba +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4998e6824733bd3f7a36dc2ccceefda5a875083 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bfe23e6eb60170c5f9f75e19f8247102a5cb51a274cb31b128e0295f4492482 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..3fa076e0d61e66ab49ad7393ded2166dc834b4d3 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c6bab8cc63969fa127be0f034efc804c532b69cc2fad979359d76f4b0876fdc +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..bae23b0dc47093c13c8a4038aaf95064c47c5b6c --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd1b41090682534e2879ea0e2dacce9c629d493a07a1615b31004690a9d74f03 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_002362_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3416d8745ff936a95f088f254d0383ee338a4c1 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3b6033cc6d317b1a2b337e91b1cc3e4f73250f88c627e29b8da229889932196 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/config.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b3c59c13a7e460dc08692ec39e251aa67142b577 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/config.json @@ -0,0 +1,68 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "current-code-baseline", + "seed42" + ] + }, + "config_fingerprint": "b389d34fe603d8f3", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1" +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/evals/samples.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..d687336b1d8afcecc8fb3487cd32781867dbbc41 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the French nation, and the capital of the French nation, and" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the same as the symbol of silver, and the symbol of gold is the same" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the same day will be Saturday, and the same day will be" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as the opposite of cold, and the same as the opposite of hot" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: 1. The sun, 2. The moon, 3. The" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a deep blue, with a deep blue border, and a deep blue border." + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the same as x, and x is the same as x, and x is" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.--S. FLETCHER LONDON, AND A NEW STUDY FOR THE SERMONS AND SERMONS, LECTURES, AND SERMONS, X-31.\n\nReferred to the Clergy of Gloucester Fair Art Training School Clauses\n\nS F. FAITHFULLY anticipating the positions which they are to occupy.-now that Dunning, Dean of Westminster, has removed from their side, namely, to Advent Sunday-School, and did even more for the cause of Education, the Burgesses have more openly expressed a wish to remain obscure.-Walker's Per.\n\nLamens, p. xvii.\n\nAfter he called on me at the University", + "<|bos|>PREFACE.\n\nTHE LIBRARY OF ACOPYRIGHT AND FIRST CONTINENT INST.\n\nM.A.\n\nCONTENTS.\n\nxix\n\nastes forlorn deliver\n\nthe fear or dan- trance to the work of ve- ners.\n\nfor it under the w 62 artist's motive lies; he gave a hand couric Woman the first couple- one leaf, weeded two other- tyred five-teixalys with emphasis. or that by friendship made two-cywork, no hand guards; the precious names Minister hand method technical review of machinery, all the idiot's necessary assistants\n\nThe handbook of defenceominal arguments over means in my", + "<|bos|>AUXE\n\nDISCOURSE XII.\n\n\"Gray Dyfe's \"Primer of Poetry.\n\nHERP, I-XXVI. \"The Marquis de Faug\u00e8res, who made the longest ode of verse\n\nApril 10. Four Years. \"LETTER XIII.\n\n155 William X. to His Sister, as it was sold at St. Petersburg, (?) in 1837. \"DEAR JOHN,-do you, I, mean to advise me through your father and your cousin Robert, how much moral and spiritual worth the person of Joseph Bonaparte may make upon him? that great intellect in England is as dear to me as", + "<|bos|> ministered, 350, 353, 355; marvelled at it till it stirred up heartfelt gratitude, 353; at\n\nMerton in the East, 40, 41, 42, 51; Burbow, the minister of the cross, 41 depicted as one whose religio powers harmonized with 35 Entertainment of this symbolism means, as I believe it would tend to debauch the thoughts and organize tendencies of a people to sin, and furnish an environment consequently personal to itself, but this avers divine authority, as developed in lies;\n\n692;", + "<|bos|>CHAPTER XIII.\n\nTHE TIDE WOMEN AND THE HEART OF THE BIBLE.\n\nAr the time of our Lord's Passion, a tenant of\n\nCrowne herself, a friendly, find, was found to be alive.\n\nAnd in truth, this great lady had madd' unto her taper.\n\nFor our Lord being dead, to his sence the sole effect was, to be dead.\n\nFrom her hand have we no reason to pray.\n\nSo am I, poore, married, or dead.\n\nthank\n\nDivers times I've much doubted whether the ghost of\n\nHobany resumed her hold, either in the Church where his", + "<|bos|>PREFACE XIII, PERS. 92. Besides these and other volumes of individuals, there are a few essays dealing respectively with political, the French Revolution, and the Confederate and Bankrupt countries,1 in general thoughts, words, and ways.\n\nWhen I have to thank most indulgent Heaven, summoned by fair profit the sowing of that mind concerning every subject which is at stake, and found it in that man who needs criticise expressions, reformations, improvisements, and principles insufficient in every one of them, it is the intention of this chapter to explain the principles and all manner of civilities, whether in political and social life,", + "<|bos|>) have published a\n\nHEREDITY OF HOMESTEAD DICTION\n\nThe proper pronunciation of the title, Bundl, and its necessary 1 a ao, band, ning from b for a. for, 17 it b simply and the na = 1 m\n\n79ly arrived at, everyday are always :\n\n23! @2 o2 b\n\nisy,\n\nThe letters and dulces of the Author in this and in the other statements, are therefore more warranted in being either for m, of, 1 or for bb, then jv\u00f4vi\u00e9 for m.\n\nThe sounds o", + "<|bos|>osso caso, e qu-cell-\n\nLive out of act e assumes he at to act of vindi-neling in the ni sa which preceded v, which was not more than vi h age and contemporaneous time, yet\n\n(rule Mn 4\n\nJOHN PIERRE 209\n\nmore to multiply v that life posses so HENT INTIF\n\n1802] (he adds, al pacity, ju \u200b1 n 227 25 194 \u664a\u200b\u9ac3 \u200b01 z\n\nSecondly, command your temper with love let the nag in to hehey n" + ] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/evals/val_bpb.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..d3763de408301bea470a103fcb6391150ed31b45 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/evals/val_bpb.json @@ -0,0 +1,94 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val_per_position": [ + { + "start": 0, + "end": 256, + "bpb": 1.1280390202815782 + }, + { + "start": 256, + "end": 512, + "bpb": 1.0668321602380306 + }, + { + "start": 512, + "end": 768, + "bpb": 1.0538636279911089 + }, + { + "start": 768, + "end": 1024, + "bpb": 1.0433245295237108 + }, + { + "start": 1024, + "end": 1280, + "bpb": 1.037325000637822 + }, + { + "start": 1280, + "end": 1536, + "bpb": 1.035617094387613 + }, + { + "start": 1536, + "end": 1792, + "bpb": 1.031107901633012 + }, + { + "start": 1792, + "end": 2048, + "bpb": 1.0284495007735763 + }, + { + "start": 2048, + "end": 2304, + "bpb": 1.0252337585887044 + }, + { + "start": 2304, + "end": 2560, + "bpb": 1.0209599977356112 + }, + { + "start": 2560, + "end": 2816, + "bpb": 1.0234279467336143 + }, + { + "start": 2816, + "end": 3072, + "bpb": 1.0242444945017173 + }, + { + "start": 3072, + "end": 3328, + "bpb": 1.0204794517385218 + }, + { + "start": 3328, + "end": 3584, + "bpb": 1.018473960788501 + }, + { + "start": 3584, + "end": 3840, + "bpb": 1.0170625289877553 + }, + { + "start": 3840, + "end": 4096, + "bpb": 1.0161504365408973 + } + ], + "val": 1.0369184312160626 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/run.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/run.json new file mode 100644 index 0000000000000000000000000000000000000000..0baf310f643e09c246106b078517949b09ce1d8f --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "branch_parent_step": null, + "config_fingerprint": "b389d34fe603d8f3", + "wandb_run_id": "dd182242", + "created_at": 1785980999 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/summary.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..850af5a3b6a28380ae4c64fb73f1ead6d0bc5e1c --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/summary.json @@ -0,0 +1,70 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset-clean-1930s", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.0917705486702252, + "minimum_sampled_val_bpb": 1.0917705486702252, + "full_val_bpb": 1.0369184312160626, + "core_metric": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the French nation, and the capital of the French nation, and" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the same as the symbol of silver, and the symbol of gold is the same" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the same day will be Saturday, and the same day will be" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as the opposite of cold, and the same as the opposite of hot" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: 1. The sun, 2. The moon, 3. The" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a deep blue, with a deep blue border, and a deep blue border." + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the same as x, and x is the same as x, and x is" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.--S. FLETCHER LONDON, AND A NEW STUDY FOR THE SERMONS AND SERMONS, LECTURES, AND SERMONS, X-31.\n\nReferred to the Clergy of Gloucester Fair Art Training School Clauses\n\nS F. FAITHFULLY anticipating the positions which they are to occupy.-now that Dunning, Dean of Westminster, has removed from their side, namely, to Advent Sunday-School, and did even more for the cause of Education, the Burgesses have more openly expressed a wish to remain obscure.-Walker's Per.\n\nLamens, p. xvii.\n\nAfter he called on me at the University", + "<|bos|>PREFACE.\n\nTHE LIBRARY OF ACOPYRIGHT AND FIRST CONTINENT INST.\n\nM.A.\n\nCONTENTS.\n\nxix\n\nastes forlorn deliver\n\nthe fear or dan- trance to the work of ve- ners.\n\nfor it under the w 62 artist's motive lies; he gave a hand couric Woman the first couple- one leaf, weeded two other- tyred five-teixalys with emphasis. or that by friendship made two-cywork, no hand guards; the precious names Minister hand method technical review of machinery, all the idiot's necessary assistants\n\nThe handbook of defenceominal arguments over means in my", + "<|bos|>AUXE\n\nDISCOURSE XII.\n\n\"Gray Dyfe's \"Primer of Poetry.\n\nHERP, I-XXVI. \"The Marquis de Faug\u00e8res, who made the longest ode of verse\n\nApril 10. Four Years. \"LETTER XIII.\n\n155 William X. to His Sister, as it was sold at St. Petersburg, (?) in 1837. \"DEAR JOHN,-do you, I, mean to advise me through your father and your cousin Robert, how much moral and spiritual worth the person of Joseph Bonaparte may make upon him? that great intellect in England is as dear to me as", + "<|bos|> ministered, 350, 353, 355; marvelled at it till it stirred up heartfelt gratitude, 353; at\n\nMerton in the East, 40, 41, 42, 51; Burbow, the minister of the cross, 41 depicted as one whose religio powers harmonized with 35 Entertainment of this symbolism means, as I believe it would tend to debauch the thoughts and organize tendencies of a people to sin, and furnish an environment consequently personal to itself, but this avers divine authority, as developed in lies;\n\n692;", + "<|bos|>CHAPTER XIII.\n\nTHE TIDE WOMEN AND THE HEART OF THE BIBLE.\n\nAr the time of our Lord's Passion, a tenant of\n\nCrowne herself, a friendly, find, was found to be alive.\n\nAnd in truth, this great lady had madd' unto her taper.\n\nFor our Lord being dead, to his sence the sole effect was, to be dead.\n\nFrom her hand have we no reason to pray.\n\nSo am I, poore, married, or dead.\n\nthank\n\nDivers times I've much doubted whether the ghost of\n\nHobany resumed her hold, either in the Church where his", + "<|bos|>PREFACE XIII, PERS. 92. Besides these and other volumes of individuals, there are a few essays dealing respectively with political, the French Revolution, and the Confederate and Bankrupt countries,1 in general thoughts, words, and ways.\n\nWhen I have to thank most indulgent Heaven, summoned by fair profit the sowing of that mind concerning every subject which is at stake, and found it in that man who needs criticise expressions, reformations, improvisements, and principles insufficient in every one of them, it is the intention of this chapter to explain the principles and all manner of civilities, whether in political and social life,", + "<|bos|>) have published a\n\nHEREDITY OF HOMESTEAD DICTION\n\nThe proper pronunciation of the title, Bundl, and its necessary 1 a ao, band, ning from b for a. for, 17 it b simply and the na = 1 m\n\n79ly arrived at, everyday are always :\n\n23! @2 o2 b\n\nisy,\n\nThe letters and dulces of the Author in this and in the other statements, are therefore more warranted in being either for m, of, 1 or for bb, then jv\u00f4vi\u00e9 for m.\n\nThe sounds o", + "<|bos|>osso caso, e qu-cell-\n\nLive out of act e assumes he at to act of vindi-neling in the ni sa which preceded v, which was not more than vi h age and contemporaneous time, yet\n\n(rule Mn 4\n\nJOHN PIERRE 209\n\nmore to multiply v that life posses so HENT INTIF\n\n1802] (he adds, al pacity, ju \u200b1 n 227 25 194 \u664a\u200b\u9ac3 \u200b01 z\n\nSecondly, command your temper with love let the nag in to hehey n" + ], + "training_time_seconds": 2532.0307886600494, + "stage_training_flops": 1.06349373718895e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.06349373718895e+18, + "config_fingerprint": "b389d34fe603d8f3", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/dd182242", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1", + "dataset_fingerprint": "0db4605cfe3a7eac", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "unique_train_tokens": 9020437955, + "effective_epochs": 0.13728471524085772 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 200, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 1000000000000, + "doc_cap": 1000000000, + "vocab_size": 32768 + }, + "created_at": 1784129374 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2 +size 132649 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-baseline-s42-v1/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f +size 410542 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..4ae73aba61297efa298d5226aaeac983393a7f78 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_000500.json @@ -0,0 +1,163 @@ +{ + "step": 500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.3446261878698493, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "wandb_run_id": "08927e2a", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,muon-momentum-constant,muon-momentum-0.83,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.83, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "muon_momentum": 0.83, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "muon-momentum-constant", + "muon-momentum-0.83", + "seed42" + ] + }, + "config_fingerprint": "6d783d8df5a5d3a0", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6d783d8df5a5d3a0" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3446261878698493, + "smooth_train_loss": 3.683517498769485, + "total_training_time": 528.0507395267487, + "stage_start_step": 0, + "stage_training_flops": 225125685264384000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2.25125685264384e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..6a5dd6ac5080c5f0a84756842961992d3006e73e --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_001000.json @@ -0,0 +1,163 @@ +{ + "step": 1000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.2473395243127656, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "wandb_run_id": "08927e2a", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,muon-momentum-constant,muon-momentum-0.83,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.83, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "muon_momentum": 0.83, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "muon-momentum-constant", + "muon-momentum-0.83", + "seed42" + ] + }, + "config_fingerprint": "6d783d8df5a5d3a0", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6d783d8df5a5d3a0" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.2473395243127656, + "smooth_train_loss": 3.5313088858304695, + "total_training_time": 1067.1877615451813, + "stage_start_step": 0, + "stage_training_flops": 450251370528768000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 4.50251370528768e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..beb446bca73369ecb2eb90639ee14e5863c6d527 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_001500.json @@ -0,0 +1,163 @@ +{ + "step": 1500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.1575217408184195, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "wandb_run_id": "08927e2a", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,muon-momentum-constant,muon-momentum-0.83,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.83, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "muon_momentum": 0.83, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "muon-momentum-constant", + "muon-momentum-0.83", + "seed42" + ] + }, + "config_fingerprint": "6d783d8df5a5d3a0", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6d783d8df5a5d3a0" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.1575217408184195, + "smooth_train_loss": 3.4231843039810266, + "total_training_time": 1605.6937384605408, + "stage_start_step": 0, + "stage_training_flops": 675377055793152000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 6.75377055793152e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..57f95bcd3b56532c40f1845ada521167d30709a4 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_002000.json @@ -0,0 +1,163 @@ +{ + "step": 2000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.1174934905823755, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "wandb_run_id": "08927e2a", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,muon-momentum-constant,muon-momentum-0.83,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.83, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "muon_momentum": 0.83, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "muon-momentum-constant", + "muon-momentum-0.83", + "seed42" + ] + }, + "config_fingerprint": "6d783d8df5a5d3a0", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6d783d8df5a5d3a0" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.1174934905823755, + "smooth_train_loss": 3.346263538059964, + "total_training_time": 2144.2248499393463, + "stage_start_step": 0, + "stage_training_flops": 900502741057536000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 9.00502741057536e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_002362.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..4053c55157e9d5056d1ab1c1c82a9e7833b16793 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/meta_002362.json @@ -0,0 +1,163 @@ +{ + "step": 2362, + "training_complete": true, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.0913370687101807, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "wandb_run_id": "08927e2a", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,muon-momentum-constant,muon-momentum-0.83,seed42", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.83, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "muon_momentum": 0.83, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "muon-momentum-constant", + "muon-momentum-0.83", + "seed42" + ] + }, + "config_fingerprint": "6d783d8df5a5d3a0", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6d783d8df5a5d3a0" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38438817, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38438817 + }, + "loop_state": { + "min_val_bpb": 1.0913370687101807, + "smooth_train_loss": 3.136666011396354, + "total_training_time": 2534.250967502594, + "stage_start_step": 0, + "stage_training_flops": 1063493737188950016, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.06349373718895e+18 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5e0ecabcbe3c4bd0d73d906bd93df277b32c92a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b53a5c5f9543b6337cbabcb983db93ed3dc73144c90146df5dc62cd0ebd92958 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..23147c3188cefa31a26e2bf5a594caa019e883ec --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a68e0cdacb0b2497ddaeb75eab3ab80372dc4aea15f8f3a4046da09e7d0423d +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7108adbdebbe71b94de29c7174eb3fb4bde0b27 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:683f0db11335666a8a73d45ff860d8eca8c5a2e9caf16abfc4192c558c95d1a9 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..250ac6dc37a87a828926f5aa20f0cbb68e368f89 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a955a670ce2a6d902494c888f30da40a73df36a603fd749de6398a059bb35dc0 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_002362.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..320db8271cdeb296aaef6dfdd7e2c9844c7ba4b3 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c6547bcdae024f741fe2e223b8e27141446d9628c495e142905b2b1f8eae032 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..69bcef5f8669f3e050ec4dce266218c290d6511d --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbd23df0af9fca80c0cbbf51e2c7ece6045fc57c8a2df02bd3c5033860ace893 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7507b1cf632b8a0d2ba08569a01d19370ccfa77 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfd24ba038952c67670b4eb79e249d407b4a0414a830ef8495eab9c9990da35b +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..94dc7d4224826095bec102e4447cd3eacf53dfb3 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ba8145bd102830452cbdf48cbb6fe2f6e153fac651c6c284ab6643f2f40364e +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d4abc56981a46fd10fe9aa23db15b14ced6f827 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c85b345f64bbe486811f973197cb0449f8110d6f6a8f1df6b5bedc1470ca5d35 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_002362_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ba4391e12714f2d6c38751a1336805cddd31694 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1cf17f6f14c8ec51d138f99bb6b7f4a5f94cb972efca14f22c37b3cd1c86831 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/config.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7cec9432adf457e9c8dfb2d49ae23a2b43f145fc --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/config.json @@ -0,0 +1,70 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "muon_momentum": 0.83, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "muon-momentum-constant", + "muon-momentum-0.83", + "seed42" + ] + }, + "config_fingerprint": "6d783d8df5a5d3a0", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1" +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/evals/samples.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..709cf09976319c8d5d76366f57241091b7a4c7cf --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the country, and the capital of the country is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the earth. The gold of the earth is the" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and so on. The\n\nSunday will be Saturday, and so on" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is hot, and hot is hot, and hot is hot, and hot is hot" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: (1) the sun, (2) the moon, (3) the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a deep blue, and the color of the color is a deep blue. The" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the 5th, and the 5th, and the" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay treats of those emotions, which may be considered as the characteristic of man's own nature, regarded as his own genius, the influence and influence of Rousseau and the examples of men who succeeded him. Climbing in the career of physical science, anticipating the voice of the heart even in this degenerate age, seeing the divine mission to which they had been set-to seek divine influences around them, thus picturing the greatness of a great man, Rousseau discovered the fullness of the universe expanding beneath him, adding a new creation to its own wisdom.\n\nFor his purpose, this volume, unlike the earlier ones of the present essay,", + "<|bos|>PREFACE. REV. CER\u043f\u0435\u043c. Various editions, for which I thank the reader, and for special reasons, especially because they are the first of class editions after the Roman.\n\nIn choosing choice editions it has never been thought my duty to give the reader a fresh idea of the time, place, and order of every first or principal place or portion of a celebrated public library, and to retain but a partial acquaintance with the French, Romans, and German Universities. Public Libraries are Ministerial libraries, reviewers, honorary authors, and filllemensensible louvbers of established Universities. Guariumius is my", + "<|bos|>RID vision of the cairn is large and rapid; when it begins to move towards the left bank it suddenly becomes obscure; at a little distance it is almost imperceptible. Spite of the extreme caution, however, of the parties, it contains always first the low sandstone cisonpeated by the mosses around the cliffs, and when it slowly advances from a (?) stone-like and depressed formation, rapidly effervesces with fresh water, which increases the lustre of the surface and gives it a majestic aspect. As it reaches higher the sandstone cisonpeated by the circle changes its state, and falls away in a very long period", + "<|bos|> ministered to get him a spirit of idealism. Historical reading one he did not wish to endorse.\n\nLondon March 11, A. S. P. 466. lect in them : he retained even a passing interest in the \"apt to say\" of \"to die,\" \"in any faith, however weak, much less to die \". The parallels that Entertains are chiefly verbal, and the results plain enough, but not the whole. The army and the politicians are not a dramatic product, but they are curious and consequently personal relations. However immediate this might be, one circumstance gleaned from them may not be amiss", + "<|bos|>PART II PART II PART III PART III PART III PART III PART I PART III PART III PART III PART III PART III Part I PART III PART\n\n PART III PART III PART I PART I PART I PART IV PART IV PART II PART IV PART V Optitude. S' International in Dr.\n\nClark Watts' Journal for April 24-26 (1857).\n\nPart II.\n\nPART III.\n\nPart IV.\n\n(a). The influens de cantes, as remembrancer, or correspondent subject, that is as to the names and parvantages of all the above that has been resumed.\n\nThe first meeting of the two associated Societies", + "<|bos|>PREFACE this scheme is too marked. But pictures and pictures crowded into them prove the disheartened youth party spirit, and overcomes the strain of the effort of social rewith regard to the ability or failure to study struck by the seventeenth century doctors in one place.\n\nFirst among them, indeed, fair profit must be made; and the diligence employed in completing the figures was then often not in proportion to the opportunity of studying them, but it led to greater variety among the new applicants. The following quotations are representative of the kind of work done by a municipal artist, as also all others.\n\nB. Ephemera, The Pilgrims\n\nHand", + "<|bos|>INTRODUCTION have published a work on the Bible as a guide for the East, and as a proper place to lay out a careful account of it. \u00fc 1\n\nAnd in the thing itself it differs from a carefully wrought work. for the purpose of deciding who simply hand the volume to the modern convenient prepared manuscript, and who may take her text; and who striking against reverence to keep silence may lay it from sight, and may dissemble it. There can be no doubt that every Bible should be translated for use to the world of men in our own country. But it is right to retain that reserve for use, as giving no", + "<|bos|>vi\n\nPREFACE\n\nix\n\nINTRODUCTION\n\nvii\n\nHOOD AND WAR LECTURES I. MILITARY ATTACHING\n\nlingering the consideration concluded on the former, the profession of more than one religious faith and practice was desired, which, if not attended to by others, whose communication with the world could be interrupted, would perhaps plead for them the invitation attending the entrance of departed souls into the sacred fellowship, which whoever wished for, would scarcely expect to go forth to good work with a purer or better spirit than his own.\n\nGeorge Howard Baillie records the opinion of many Christians who read the word Socialist many years back, he would like" + ] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/evals/val_bpb.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..70626f7e62cdcdbf54ea6654f61044e4eb37e58d --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/evals/val_bpb.json @@ -0,0 +1,94 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val_per_position": [ + { + "start": 0, + "end": 256, + "bpb": 1.1274765564193325 + }, + { + "start": 256, + "end": 512, + "bpb": 1.0657887573320168 + }, + { + "start": 512, + "end": 768, + "bpb": 1.0529323371637673 + }, + { + "start": 768, + "end": 1024, + "bpb": 1.042417557436246 + }, + { + "start": 1024, + "end": 1280, + "bpb": 1.0361290704191266 + }, + { + "start": 1280, + "end": 1536, + "bpb": 1.0340712039397197 + }, + { + "start": 1536, + "end": 1792, + "bpb": 1.0296966597181718 + }, + { + "start": 1792, + "end": 2048, + "bpb": 1.0268858353043673 + }, + { + "start": 2048, + "end": 2304, + "bpb": 1.0237218740325673 + }, + { + "start": 2304, + "end": 2560, + "bpb": 1.0197109227177406 + }, + { + "start": 2560, + "end": 2816, + "bpb": 1.02172956043314 + }, + { + "start": 2816, + "end": 3072, + "bpb": 1.0230340484873293 + }, + { + "start": 3072, + "end": 3328, + "bpb": 1.0191460973859146 + }, + { + "start": 3328, + "end": 3584, + "bpb": 1.0169676603801217 + }, + { + "start": 3584, + "end": 3840, + "bpb": 1.0154473730127702 + }, + { + "start": 3840, + "end": 4096, + "bpb": 1.014662013752163 + } + ], + "val": 1.0356202398726997 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/run.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/run.json new file mode 100644 index 0000000000000000000000000000000000000000..a24abd9b5ef67ecb346374f06cc7365b45b994bb --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "branch_parent_step": null, + "config_fingerprint": "6d783d8df5a5d3a0", + "wandb_run_id": "08927e2a", + "created_at": 1785983842 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/summary.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..701f1aa28c42f435af60bd5e9999d33d8ea5c257 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/summary.json @@ -0,0 +1,70 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset-clean-1930s", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.0913370687101807, + "minimum_sampled_val_bpb": 1.0913370687101807, + "full_val_bpb": 1.0356202398726997, + "core_metric": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the country, and the capital of the country is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the earth. The gold of the earth is the" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and so on. The\n\nSunday will be Saturday, and so on" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is hot, and hot is hot, and hot is hot, and hot is hot" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: (1) the sun, (2) the moon, (3) the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a deep blue, and the color of the color is a deep blue. The" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the 5th, and the 5th, and the" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay treats of those emotions, which may be considered as the characteristic of man's own nature, regarded as his own genius, the influence and influence of Rousseau and the examples of men who succeeded him. Climbing in the career of physical science, anticipating the voice of the heart even in this degenerate age, seeing the divine mission to which they had been set-to seek divine influences around them, thus picturing the greatness of a great man, Rousseau discovered the fullness of the universe expanding beneath him, adding a new creation to its own wisdom.\n\nFor his purpose, this volume, unlike the earlier ones of the present essay,", + "<|bos|>PREFACE. REV. CER\u043f\u0435\u043c. Various editions, for which I thank the reader, and for special reasons, especially because they are the first of class editions after the Roman.\n\nIn choosing choice editions it has never been thought my duty to give the reader a fresh idea of the time, place, and order of every first or principal place or portion of a celebrated public library, and to retain but a partial acquaintance with the French, Romans, and German Universities. Public Libraries are Ministerial libraries, reviewers, honorary authors, and filllemensensible louvbers of established Universities. Guariumius is my", + "<|bos|>RID vision of the cairn is large and rapid; when it begins to move towards the left bank it suddenly becomes obscure; at a little distance it is almost imperceptible. Spite of the extreme caution, however, of the parties, it contains always first the low sandstone cisonpeated by the mosses around the cliffs, and when it slowly advances from a (?) stone-like and depressed formation, rapidly effervesces with fresh water, which increases the lustre of the surface and gives it a majestic aspect. As it reaches higher the sandstone cisonpeated by the circle changes its state, and falls away in a very long period", + "<|bos|> ministered to get him a spirit of idealism. Historical reading one he did not wish to endorse.\n\nLondon March 11, A. S. P. 466. lect in them : he retained even a passing interest in the \"apt to say\" of \"to die,\" \"in any faith, however weak, much less to die \". The parallels that Entertains are chiefly verbal, and the results plain enough, but not the whole. The army and the politicians are not a dramatic product, but they are curious and consequently personal relations. However immediate this might be, one circumstance gleaned from them may not be amiss", + "<|bos|>PART II PART II PART III PART III PART III PART III PART I PART III PART III PART III PART III PART III Part I PART III PART\n\n PART III PART III PART I PART I PART I PART IV PART IV PART II PART IV PART V Optitude. S' International in Dr.\n\nClark Watts' Journal for April 24-26 (1857).\n\nPart II.\n\nPART III.\n\nPart IV.\n\n(a). The influens de cantes, as remembrancer, or correspondent subject, that is as to the names and parvantages of all the above that has been resumed.\n\nThe first meeting of the two associated Societies", + "<|bos|>PREFACE this scheme is too marked. But pictures and pictures crowded into them prove the disheartened youth party spirit, and overcomes the strain of the effort of social rewith regard to the ability or failure to study struck by the seventeenth century doctors in one place.\n\nFirst among them, indeed, fair profit must be made; and the diligence employed in completing the figures was then often not in proportion to the opportunity of studying them, but it led to greater variety among the new applicants. The following quotations are representative of the kind of work done by a municipal artist, as also all others.\n\nB. Ephemera, The Pilgrims\n\nHand", + "<|bos|>INTRODUCTION have published a work on the Bible as a guide for the East, and as a proper place to lay out a careful account of it. \u00fc 1\n\nAnd in the thing itself it differs from a carefully wrought work. for the purpose of deciding who simply hand the volume to the modern convenient prepared manuscript, and who may take her text; and who striking against reverence to keep silence may lay it from sight, and may dissemble it. There can be no doubt that every Bible should be translated for use to the world of men in our own country. But it is right to retain that reserve for use, as giving no", + "<|bos|>vi\n\nPREFACE\n\nix\n\nINTRODUCTION\n\nvii\n\nHOOD AND WAR LECTURES I. MILITARY ATTACHING\n\nlingering the consideration concluded on the former, the profession of more than one religious faith and practice was desired, which, if not attended to by others, whose communication with the world could be interrupted, would perhaps plead for them the invitation attending the entrance of departed souls into the sacred fellowship, which whoever wished for, would scarcely expect to go forth to good work with a purer or better spirit than his own.\n\nGeorge Howard Baillie records the opinion of many Christians who read the word Socialist many years back, he would like" + ], + "training_time_seconds": 2534.250967502594, + "stage_training_flops": 1.06349373718895e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.06349373718895e+18, + "config_fingerprint": "6d783d8df5a5d3a0", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/08927e2a", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1", + "dataset_fingerprint": "0db4605cfe3a7eac", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "unique_train_tokens": 9020437955, + "effective_epochs": 0.13728471524085772 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 200, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 1000000000000, + "doc_cap": 1000000000, + "vocab_size": 32768 + }, + "created_at": 1784129374 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2 +size 132649 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon083-s42-v1/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f +size 410542 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..8553a3938d1a650b2cb8f4f76a321b674d888b92 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_000500.json @@ -0,0 +1,163 @@ +{ + "step": 500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.3401702205232602, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "wandb_run_id": "0271e3eb", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,seed42,muon-momentum-constant,muon-momentum-0.90", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "seed42", + "muon-momentum-constant", + "muon-momentum-0.90" + ] + }, + "config_fingerprint": "6a707178cf2e40fc", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6a707178cf2e40fc" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3401702205232602, + "smooth_train_loss": 3.6873101553428147, + "total_training_time": 526.4253585338593, + "stage_start_step": 0, + "stage_training_flops": 225125685264384000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2.25125685264384e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..6289744e323742e1444f59af60cfa6aedd70c708 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_001000.json @@ -0,0 +1,163 @@ +{ + "step": 1000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.236094953958828, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "wandb_run_id": "0271e3eb", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,seed42,muon-momentum-constant,muon-momentum-0.90", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "seed42", + "muon-momentum-constant", + "muon-momentum-0.90" + ] + }, + "config_fingerprint": "6a707178cf2e40fc", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6a707178cf2e40fc" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.236094953958828, + "smooth_train_loss": 3.5190564919373246, + "total_training_time": 1064.2983202934265, + "stage_start_step": 0, + "stage_training_flops": 450251370528768000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 4.50251370528768e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..ba6dc1521c57f84aa0ddc7efb45f4815ed7fe1e6 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_001500.json @@ -0,0 +1,163 @@ +{ + "step": 1500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.1537679671738093, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "wandb_run_id": "0271e3eb", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,seed42,muon-momentum-constant,muon-momentum-0.90", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "seed42", + "muon-momentum-constant", + "muon-momentum-0.90" + ] + }, + "config_fingerprint": "6a707178cf2e40fc", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6a707178cf2e40fc" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.1537679671738093, + "smooth_train_loss": 3.4145766089682157, + "total_training_time": 1601.6644263267517, + "stage_start_step": 0, + "stage_training_flops": 675377055793152000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 6.75377055793152e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..bcdd4ffc0f159b22d7e12941cb081a7eea0b16b3 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_002000.json @@ -0,0 +1,163 @@ +{ + "step": 2000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.1151770575853448, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "wandb_run_id": "0271e3eb", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,seed42,muon-momentum-constant,muon-momentum-0.90", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "seed42", + "muon-momentum-constant", + "muon-momentum-0.90" + ] + }, + "config_fingerprint": "6a707178cf2e40fc", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6a707178cf2e40fc" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.1151770575853448, + "smooth_train_loss": 3.3391174163554944, + "total_training_time": 2139.0777184963226, + "stage_start_step": 0, + "stage_training_flops": 900502741057536000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 9.00502741057536e+17 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_002362.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..aa78c7825c4afc73c5a3d45c96a39a1a3afb392c --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/meta_002362.json @@ -0,0 +1,163 @@ +{ + "step": 2362, + "training_complete": true, + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "val_bpb": 1.089106110634849, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "SSSL" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "wandb_run_id": "0271e3eb", + "wandb_group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,ctx4096,sssl,fulltok,fp8,h100,autoresearch-transfer-v1,seed42,muon-momentum-constant,muon-momentum-0.90", + "device_type": "", + "fp8": true, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "muon_momentum": 0.9, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "init_from_checkpoint_dir": null, + "init_from_step": -1, + "no_init_optimizer": false, + "branch_lr_schedule": "branch", + "branch_parent_experiment_id": null, + "pretokenized": true, + "data_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/data", + "tokenizer_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer", + "pretokenized_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/pretok", + "mixture_source_dirs": null, + "checkpoint_dir": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment_config": "/workspace/nanochat/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/config.json", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "seed42", + "muon-momentum-constant", + "muon-momentum-0.90" + ] + }, + "config_fingerprint": "6a707178cf2e40fc", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6a707178cf2e40fc" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38438817, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38438817 + }, + "loop_state": { + "min_val_bpb": 1.089106110634849, + "smooth_train_loss": 3.1315954124792693, + "total_training_time": 2528.233864545822, + "stage_start_step": 0, + "stage_training_flops": 1063493737188950016, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.06349373718895e+18 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..e96b21b8cafaf9b211c049b0c1465e3368500c12 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68a44c5037e6cd6fe20c5dbdbb981a1a468459d884390572ac977439de7ddd69 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..d459c86bf922e749631f2c88de578952e8e5ccf5 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:750728d13176232157adda742e0e3039391d7d6dab69e1faf0c3aa42d3929e96 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbd62c0b1768f1a88998e923037eb2f6101561a1 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da509d33e20806bbeeeda1fd6d7a74df3e598077f515ed3b982deff299256a00 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6457e803f1588dae2fd34d76a00f953a14acf3a6 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea83b8e6f3ecfe7941edf52e14e45a901fda25832c596c22894d017493abf0ac +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_002362.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..d23dbe10842333f7a81500ab53806b914ef87648 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0702cec36e71ca2195924c97e08fb178892f13fc99bf6b51ff1f1fc774776ed +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c6092c5794dcf5fcbece2719c2c74936598eaf1 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dc26b319b73b7a4fcf04440fb6e410aef007f061f510b1bb361778fd73c4045 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..60dd06bb5b5757e52fceff40d0cce3006190df88 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f201636a64b81cd5b9a6a0a06e64e4a7939e071a3967203a6fb1e6d9f15f109 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6a3770c0b652d0066df67f138fc2f51c0993fa6 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1e5404258e1c29c15b755a2880237c9d0f1f652752876afe7b1e4299e095102 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ab8ca5af0d209e6e6b90b38f8e62fb06113b217 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c18d566e9183752553658da01b82ca6469c1c47447258c72558676e3d82942a0 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_002362_rank0.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..64ae5ece1b97ff71922db143810e3a0439d17e9f --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:219cad4a6369b79f19b15148bfcf23f4fcf78bf5e1da02895e7f8197db185b5b +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/config.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..948aa9c1733fb0aea0b860e24a9b2b43ac6c3f07 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/config.json @@ -0,0 +1,70 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 180, + "download_workers": 4 + }, + "tokenizer": { + "mode": "reuse", + "source_experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "muon_momentum": 0.9, + "device_batch_size": 8, + "total_batch_size": 524288, + "fp8": true, + "fp8_recipe": "tensorwise", + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano", + "keep_local_checkpoints": 1 + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "group": "clean1930s-d12-ctx4096-autoresearch-transfer-v1", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "ctx4096", + "sssl", + "fulltok", + "fp8", + "h100", + "autoresearch-transfer-v1", + "seed42", + "muon-momentum-constant", + "muon-momentum-0.90" + ] + }, + "config_fingerprint": "6a707178cf2e40fc", + "artifact_path": "experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1" +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/evals/samples.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..fc61fe6bb65a08d338d86949c2163784aac595e2 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the country, and the capital of the country is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the gold of the sun, and the symbol of the sun is the gold of" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. The\n\nLord's day is the day of the Lord's, and" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the hot of the sun, and the opposite of cold is the hot of the" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a white one, and I have a white one. I have a white one" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the same as x, and the same as x, and the same as x" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay was contributed to the \"Introduction \"-volume journal of the \"Society for the Promotion of\n\nScience,\" which has always prepared it for use only in the \"training Fair.\" Page 303\n\nSENTIMENTS AND INDUSTRIAL ENDS.\n\nWEALTH-ONE OATH.-Old woman Dunning called divine service to mind if the Old woman was mazy, if she was a little old man.\n\nHis servitor, Billy Freykin, had given a bonne bouche to the palate of the young chemist.\n\nThe fine old lady came about the first of each month of the year and adored", + "<|bos|>PREFACE.\n\nTHE LIBRARY OF ACOPYRIGHT AND VOLUME III.\n\nThe Proceedings for the Percussion-Capric Outastes.\n\nTHE Volume, which has been the first to be issued after the studies prosecuted, was founded during the year 1871, when the University Government assigned the Library of the Biological Woman to the two rooms of the first sewing society.\n\nUpon the turning over of the little volume, with its completest arrangement, made in 1873, no further contributions were made. Still, the method technical review of the volume is to be found necessary.\n\nSome inferior articles, merely established as stores of overwork, allowing", + "<|bos|>The vision of the man who is lord of the brute Dyfe, also brings the vision directly into collision with Vanderburgh's doctrine as stated above. It corresponded more exactly in form and structure to his views, than to those of the Edwards. The vision first introduces the man cison\u00e1red by the destroyed horse Dyfe as a homoeoteer to life, while in another and more elaborate form he escapes but leaves the precipice. Sometimes the vision has the rarity of its object, sometimes a foreign element appears and notes it. It was made where the rarity of cancelation appears to be strongest. The facts exhibit only the", + "<|bos|>ESLIE 350, 37 Ind. 125; Martel, XIV, 703; Notes on Aphrodoclus\n\nMut\u00f3s, 765; Deshon, 756; Quaere, de Ecclesia die S.J...B. .954; Religio: Pav. Theol., v, i, xii, xiv; [Recond. Am. Soc. Antiq. Society [1905], iv 3, 30; Urr. Arch. Antiq., vii, vii; a note follows, this allusion.-Cf.\n\nBeb. ", + "<|bos|>CHAPTER XIII.\n\nTHE TIDE OF MOUNTAINS, OR THE ROO- NUM AND THE CROSS CHURCHES-TI- TIONAL LATTER-ROADS, OR UNIFORM-ROPS IN ALGIANTRA-GAT. AF'LIGIOUS IMPRESSIONS OF THE MAJOR DESER- MAUR.\n\nNUM is manifest on many a bright day; in others no brilliancy can be feared. The pale shades which show on the mountains are subject to the most violent tempests, and the wild animals in the wood are alarmed that they may leap out of darkness, and the gorgeous insect cover", + "<|bos|>PREFACE\n\n771. On this I can wish and was glad to pass my whole life, for it was part of my education: at least, I was intimately able to appreciate the advantages of education and to study it by heart; the only thing, I say, that can, ever, make me feel an eager desire to study every subject which is at present open and engrosses my time. While I criticise my arrangement of my studies, I don't find any principles insufficient to insure my success.s In the same manner, I am a pretty strong advocate for the necessity of a little real, lasting preparation for a profession, where", + "<|bos|>INTRODUCTION.\n\nThe Sultan's Architect has prepared the Royal Equestrian School for the Service of Devotion; and there were thirteen\n\nlarge chests of Stationers' Hallowed Schools, without regard to local or political traditions, not a few of which are excellent, simply because the Council made themselves modern, prepared exactly by the school authorities, and always gifted as with the eyes and ears of the people. These Schools were, at the Time of which I write, consigned to an excellent school; but before their return to Graduates, twenty-four lines and seven black blackbirds were on toberage around them. All buildings that could", + "<|bos|>vi\n\nof 1889-90-\n\nLive out of doors in Yucatan.\n\nTalk of the Indies in the reign concluded on by Count Mochilla in his letter to the Czar and presented to Congress at\n\nPhiladelphia by M. Julien.\n\n\"It is not impossible that some mode of supplying us with necessaries may be suggested by these colonies.\"\n\nDear Brother M. has sent you a sketch of his history with a Life of Manila. It is the good fortune of a minister to be a mother to this people. It is of importance to the interests of Christians in the sugar-growing districts that Fernando, he tells us" + ] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/evals/val_bpb.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..451c12f4ea8609d5db51255518a0b6d4294b8bc1 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/evals/val_bpb.json @@ -0,0 +1,94 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val_per_position": [ + { + "start": 0, + "end": 256, + "bpb": 1.1253436362626366 + }, + { + "start": 256, + "end": 512, + "bpb": 1.0642184743188667 + }, + { + "start": 512, + "end": 768, + "bpb": 1.0510683484355177 + }, + { + "start": 768, + "end": 1024, + "bpb": 1.0405059216393195 + }, + { + "start": 1024, + "end": 1280, + "bpb": 1.0345590529216036 + }, + { + "start": 1280, + "end": 1536, + "bpb": 1.0328886578039924 + }, + { + "start": 1536, + "end": 1792, + "bpb": 1.0281508738706628 + }, + { + "start": 1792, + "end": 2048, + "bpb": 1.0252040319043694 + }, + { + "start": 2048, + "end": 2304, + "bpb": 1.0220662501454005 + }, + { + "start": 2304, + "end": 2560, + "bpb": 1.018049082689113 + }, + { + "start": 2560, + "end": 2816, + "bpb": 1.0200546960682964 + }, + { + "start": 2816, + "end": 3072, + "bpb": 1.021086121097632 + }, + { + "start": 3072, + "end": 3328, + "bpb": 1.017408884359891 + }, + { + "start": 3328, + "end": 3584, + "bpb": 1.0154681928589606 + }, + { + "start": 3584, + "end": 3840, + "bpb": 1.0142038984329307 + }, + { + "start": 3840, + "end": 4096, + "bpb": 1.012878088694088 + } + ], + "val": 1.03395383030625 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/run.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/run.json new file mode 100644 index 0000000000000000000000000000000000000000..ac02d165a30e0be166928c434010278fd50ad9f0 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/run.json @@ -0,0 +1,11 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "branch_parent_step": null, + "config_fingerprint": "6a707178cf2e40fc", + "wandb_run_id": "0271e3eb", + "created_at": 1785986530 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/summary.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2bfe002379e8e51ee7eb389306c38346d6af8dcf --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/summary.json @@ -0,0 +1,70 @@ +{ + "experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset-clean-1930s", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.089106110634849, + "minimum_sampled_val_bpb": 1.089106110634849, + "full_val_bpb": 1.03395383030625, + "core_metric": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the country, and the capital of the country is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the gold of the sun, and the symbol of the sun is the gold of" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. The\n\nLord's day is the day of the Lord's, and" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the hot of the sun, and the opposite of cold is the hot of the" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a white one, and I have a white one. I have a white one" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the same as x, and the same as x, and the same as x" + } + ], + "unconditioned_samples": [ + "<|bos|>PREFACE.\n\nTHIS essay was contributed to the \"Introduction \"-volume journal of the \"Society for the Promotion of\n\nScience,\" which has always prepared it for use only in the \"training Fair.\" Page 303\n\nSENTIMENTS AND INDUSTRIAL ENDS.\n\nWEALTH-ONE OATH.-Old woman Dunning called divine service to mind if the Old woman was mazy, if she was a little old man.\n\nHis servitor, Billy Freykin, had given a bonne bouche to the palate of the young chemist.\n\nThe fine old lady came about the first of each month of the year and adored", + "<|bos|>PREFACE.\n\nTHE LIBRARY OF ACOPYRIGHT AND VOLUME III.\n\nThe Proceedings for the Percussion-Capric Outastes.\n\nTHE Volume, which has been the first to be issued after the studies prosecuted, was founded during the year 1871, when the University Government assigned the Library of the Biological Woman to the two rooms of the first sewing society.\n\nUpon the turning over of the little volume, with its completest arrangement, made in 1873, no further contributions were made. Still, the method technical review of the volume is to be found necessary.\n\nSome inferior articles, merely established as stores of overwork, allowing", + "<|bos|>The vision of the man who is lord of the brute Dyfe, also brings the vision directly into collision with Vanderburgh's doctrine as stated above. It corresponded more exactly in form and structure to his views, than to those of the Edwards. The vision first introduces the man cison\u00e1red by the destroyed horse Dyfe as a homoeoteer to life, while in another and more elaborate form he escapes but leaves the precipice. Sometimes the vision has the rarity of its object, sometimes a foreign element appears and notes it. It was made where the rarity of cancelation appears to be strongest. The facts exhibit only the", + "<|bos|>ESLIE 350, 37 Ind. 125; Martel, XIV, 703; Notes on Aphrodoclus\n\nMut\u00f3s, 765; Deshon, 756; Quaere, de Ecclesia die S.J...B. .954; Religio: Pav. Theol., v, i, xii, xiv; [Recond. Am. Soc. Antiq. Society [1905], iv 3, 30; Urr. Arch. Antiq., vii, vii; a note follows, this allusion.-Cf.\n\nBeb. ", + "<|bos|>CHAPTER XIII.\n\nTHE TIDE OF MOUNTAINS, OR THE ROO- NUM AND THE CROSS CHURCHES-TI- TIONAL LATTER-ROADS, OR UNIFORM-ROPS IN ALGIANTRA-GAT. AF'LIGIOUS IMPRESSIONS OF THE MAJOR DESER- MAUR.\n\nNUM is manifest on many a bright day; in others no brilliancy can be feared. The pale shades which show on the mountains are subject to the most violent tempests, and the wild animals in the wood are alarmed that they may leap out of darkness, and the gorgeous insect cover", + "<|bos|>PREFACE\n\n771. On this I can wish and was glad to pass my whole life, for it was part of my education: at least, I was intimately able to appreciate the advantages of education and to study it by heart; the only thing, I say, that can, ever, make me feel an eager desire to study every subject which is at present open and engrosses my time. While I criticise my arrangement of my studies, I don't find any principles insufficient to insure my success.s In the same manner, I am a pretty strong advocate for the necessity of a little real, lasting preparation for a profession, where", + "<|bos|>INTRODUCTION.\n\nThe Sultan's Architect has prepared the Royal Equestrian School for the Service of Devotion; and there were thirteen\n\nlarge chests of Stationers' Hallowed Schools, without regard to local or political traditions, not a few of which are excellent, simply because the Council made themselves modern, prepared exactly by the school authorities, and always gifted as with the eyes and ears of the people. These Schools were, at the Time of which I write, consigned to an excellent school; but before their return to Graduates, twenty-four lines and seven black blackbirds were on toberage around them. All buildings that could", + "<|bos|>vi\n\nof 1889-90-\n\nLive out of doors in Yucatan.\n\nTalk of the Indies in the reign concluded on by Count Mochilla in his letter to the Czar and presented to Congress at\n\nPhiladelphia by M. Julien.\n\n\"It is not impossible that some mode of supplying us with necessaries may be suggested by these colonies.\"\n\nDear Brother M. has sent you a sketch of his history with a Life of Manila. It is the good fortune of a minister to be a mother to this people. It is of importance to the interests of Christians in the sugar-growing districts that Fernando, he tells us" + ], + "training_time_seconds": 2528.233864545822, + "stage_training_flops": 1.06349373718895e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.06349373718895e+18, + "config_fingerprint": "6a707178cf2e40fc", + "git_commit_sha": "955f6037e852088dcc422f046ad5236ea66090b7", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/0271e3eb", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1", + "dataset_fingerprint": "0db4605cfe3a7eac", + "tokenizer_fingerprint": "21e99d5cdeeaa660", + "unique_train_tokens": 9020437955, + "effective_epochs": 0.13728471524085772 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/experiment_tokenizer.json b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..abb9ccfdfc6557194bfe6a3551bcdd320b777e78 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "clean1930s-d24-r12-ctx4096-fulltok-v1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 200, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 1000000000000, + "doc_cap": 1000000000, + "vocab_size": 32768 + }, + "created_at": 1784129374 +} diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/token_bytes.pt b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..737ab9ff9eafdbd5bfa971d0390b520b87ebb55a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc779ae25dfa6f35146f7b9991fa3bab9f2a82a89a4dd92fbd4a57655680e7e2 +size 132649 diff --git a/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/tokenizer.pkl b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..34650d2ed06bbfb645ad394f823340b08c7af1ac --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-ctx4096-sssl-fulltok-artransfer-muon090-s42-v1/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155d20e57ea2203cab333207e97b2bec8c0d224678bfce2f019cff3a8ddd940f +size 410542 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_000500.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..2b43afe320038ac431689c5ad1ca6ab1a88e82e9 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_000500.json @@ -0,0 +1,142 @@ +{ + "step": 500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "val_bpb": 1.3328434591455662, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-muoneq-muonplus", + "wandb_run_id": "95708491", + "wandb_group": "clean1930s-d12", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,muoneq,muonplus,optimizer-ablation", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/config.json", + "tokenizer_fingerprint": "803338edbf3458e6", + "git_commit_sha": "a47442614b208fd6df1549f6ca130504ca8975ed", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 26, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-muoneq-muonplus", + "group": "clean1930s-d12", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "muoneq", + "muonplus", + "optimizer-ablation" + ] + }, + "config_fingerprint": "4f01f653a56d0daf", + "artifact_path": "experiments/clean1930s-d12-r11.25-muoneq-muonplus" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "4f01f653a56d0daf" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3328434591455662, + "smooth_train_loss": 3.828870216538712, + "total_training_time": 1320.5494623184204, + "stage_training_flops": 232547388751872000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 232547388751872000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_001000.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..bdf94f480fbbec1bcd7083cfefb90bbde43c200c --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_001000.json @@ -0,0 +1,142 @@ +{ + "step": 1000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "val_bpb": 1.2431777724823778, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-muoneq-muonplus", + "wandb_run_id": "95708491", + "wandb_group": "clean1930s-d12", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,muoneq,muonplus,optimizer-ablation", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/config.json", + "tokenizer_fingerprint": "803338edbf3458e6", + "git_commit_sha": "a47442614b208fd6df1549f6ca130504ca8975ed", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 26, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-muoneq-muonplus", + "group": "clean1930s-d12", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "muoneq", + "muonplus", + "optimizer-ablation" + ] + }, + "config_fingerprint": "4f01f653a56d0daf", + "artifact_path": "experiments/clean1930s-d12-r11.25-muoneq-muonplus" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "4f01f653a56d0daf" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.2431777724823778, + "smooth_train_loss": 3.570735216236959, + "total_training_time": 2669.536954641342, + "stage_training_flops": 465094777503744000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 465094777503744000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_001500.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..befaebca8c3fc88c204c92805f9112350309e715 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_001500.json @@ -0,0 +1,142 @@ +{ + "step": 1500, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "val_bpb": 1.1772538631419072, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-muoneq-muonplus", + "wandb_run_id": "95708491", + "wandb_group": "clean1930s-d12", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,muoneq,muonplus,optimizer-ablation", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/config.json", + "tokenizer_fingerprint": "803338edbf3458e6", + "git_commit_sha": "a47442614b208fd6df1549f6ca130504ca8975ed", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 26, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-muoneq-muonplus", + "group": "clean1930s-d12", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "muoneq", + "muonplus", + "optimizer-ablation" + ] + }, + "config_fingerprint": "4f01f653a56d0daf", + "artifact_path": "experiments/clean1930s-d12-r11.25-muoneq-muonplus" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "4f01f653a56d0daf" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.1772538631419072, + "smooth_train_loss": 3.3416836335468014, + "total_training_time": 4018.106310606003, + "stage_training_flops": 697642166255616000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 697642166255616000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_002000.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..965abfb6af90aa47ff16b80373fc6a631d1dfa85 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_002000.json @@ -0,0 +1,142 @@ +{ + "step": 2000, + "training_complete": false, + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "val_bpb": 1.1372931129764448, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-muoneq-muonplus", + "wandb_run_id": "95708491", + "wandb_group": "clean1930s-d12", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,muoneq,muonplus,optimizer-ablation", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/config.json", + "tokenizer_fingerprint": "803338edbf3458e6", + "git_commit_sha": "a47442614b208fd6df1549f6ca130504ca8975ed", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 26, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-muoneq-muonplus", + "group": "clean1930s-d12", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "muoneq", + "muonplus", + "optimizer-ablation" + ] + }, + "config_fingerprint": "4f01f653a56d0daf", + "artifact_path": "experiments/clean1930s-d12-r11.25-muoneq-muonplus" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "4f01f653a56d0daf" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.1372931129764448, + "smooth_train_loss": 3.1939839740715126, + "total_training_time": 5366.212870359421, + "stage_training_flops": 930189555007488000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 930189555007488000 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_002362.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..bd2c1ae345b3e8d4d83a1e01e89124942a59b52c --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/meta_002362.json @@ -0,0 +1,142 @@ +{ + "step": 2362, + "training_complete": true, + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "val_bpb": 1.1162452479490097, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "clean1930s-d12-r11.25-muoneq-muonplus", + "wandb_run_id": "95708491", + "wandb_group": "clean1930s-d12", + "wandb_tags": "think-dataset-clean-1930s,d12,ratio11.25,muoneq,muonplus,optimizer-ablation", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment_config": "/content/nanochat_cache/experiments/clean1930s-d12-r11.25-muoneq-muonplus/config.json", + "tokenizer_fingerprint": "803338edbf3458e6", + "git_commit_sha": "a47442614b208fd6df1549f6ca130504ca8975ed", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "clean1930s-d12-r11.25-muoneq-muonplus", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 26, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-muoneq-muonplus", + "group": "clean1930s-d12", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "muoneq", + "muonplus", + "optimizer-ablation" + ] + }, + "config_fingerprint": "4f01f653a56d0daf", + "artifact_path": "experiments/clean1930s-d12-r11.25-muoneq-muonplus" + }, + "stage": "base", + "base_experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "4f01f653a56d0daf" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38438817, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38438817 + }, + "loop_state": { + "min_val_bpb": 1.1162452479490097, + "smooth_train_loss": 3.1283991602458974, + "total_training_time": 6341.746661424637, + "stage_training_flops": 1098553864463843328, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1098553864463843328 + } +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_000500.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7d0b4d7847fd743553b74033bd14307cf36b4f2 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49648b40dd6946856f8114d5e6ed78d33f45f1381c17766661de01e2e9eba2bb +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_001000.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e6b8200a0d698fe89e1d0d9195a138ed32a77d0 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b51d3b5ece5cf94619fe0fd9dc11f956e93bba17161011c0ccda7ff307f79753 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_001500.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..84b572539150af7411800e18b74296bf1699ddb2 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6161003a46a5be804b34fc295d4586ecb29e09936953e613c5b5860fd1548309 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_002000.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c697277e1a05a98d87be429be8d4678e65e542a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb22a4f3416bc93c937ab2b12e7d8bf112bd20b2983c7f145c5eadbc6bf16047 +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_002362.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..b39f67fa282e401db0de0567e32830568871cd56 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6b0a5ce017e532265d1fcb329dd0f8051e752ff754c00669892033553ebb7e +size 792761690 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_000500_rank0.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc17ecd8d93252d804b7553fa5b06299fc4e9d4a --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ffde69c12ed9c8c0e7058e75c4bfd74788f35ffb85ca002569e69f1d1395fca +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_001000_rank0.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..40116a6365cab40da5b220d3798ecd1f1d4b81d5 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f91b1f0394b8a49baeab03b9252226cfe89ef414b1ea4d60e54d6129100f3ce +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_001500_rank0.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d01fbfa198c4a012baaca755f4360710a059825 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d00f57df038ee139e432df8571c57c2f99ddd82d9c9fae22e6e71553ca7dc23 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_002000_rank0.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5df93e3b783934da48d931c88bccf643c644fbd5 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b9d9fb936a69f962545728bcdc659964a3786c1f242ffdbe23ff02187713a23 +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_002362_rank0.pt b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7dfad70e4cd0e9404c251644fe7150f5365cf69 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa420d24fb27abbcb471da9820bd2f8a05285e6862704e8a073451f0a201dd8c +size 1246165357 diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/config.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/config.json new file mode 100644 index 0000000000000000000000000000000000000000..48391fc7aa5d21c347c5dd6c95b08797b8f2bebb --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/config.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "clean1930s-d12-r11.25-muoneq-muonplus", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset-clean-1930s", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 26, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "clean1930s-d12-r11.25-muoneq-muonplus", + "group": "clean1930s-d12", + "tags": [ + "think-dataset-clean-1930s", + "d12", + "ratio11.25", + "muoneq", + "muonplus", + "optimizer-ablation" + ] + }, + "config_fingerprint": "4f01f653a56d0daf", + "artifact_path": "experiments/clean1930s-d12-r11.25-muoneq-muonplus" +} diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/core.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/core.json new file mode 100644 index 0000000000000000000000000000000000000000..b3b01c0383482064d10911a98c019693e8131adb --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/core.json @@ -0,0 +1,56 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": 0.06305335515020447, + "core_results": { + "hellaswag_zeroshot": 0.27713602781295776, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.08601938933134079, + "arc_easy": 0.32154881954193115, + "arc_challenge": 0.21587030589580536, + "copa": 0.5, + "commonsense_qa": 0.28337427973747253, + "piqa": 0.5424374341964722, + "openbook_qa": 0.23200000822544098, + "lambada_openai": 0.21191538870334625, + "hellaswag": 0.27932682633399963, + "winograd": 0.5201465487480164, + "winogrande": 0.5003946423530579, + "bigbench_dyck_languages": 0.07200000435113907, + "agi_eval_lsat_ar": 0.260869562625885, + "bigbench_cs_algorithms": 0.4371212124824524, + "bigbench_operators": 0.10476190596818924, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.020813623443245888, + "coqa": 0.07303018867969513, + "boolq": 0.5412843823432922, + "bigbench_language_identification": 0.25529998540878296 + }, + "centered_results": { + "hellaswag_zeroshot": 0.036181370417277016, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.08601938933134079, + "arc_easy": 0.0953984260559082, + "arc_challenge": -0.04550625880559286, + "copa": 0.0, + "commonsense_qa": 0.10421784967184065, + "piqa": 0.08487486839294434, + "openbook_qa": -0.02399998903274536, + "lambada_openai": 0.21191538870334625, + "hellaswag": 0.03910243511199951, + "winograd": 0.040293097496032715, + "winogrande": 0.0007892847061157227, + "bigbench_dyck_languages": 0.07200000435113907, + "agi_eval_lsat_ar": 0.07608695328235625, + "bigbench_cs_algorithms": 0.4371212124824524, + "bigbench_operators": 0.10476190596818924, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.020813623443245888, + "coqa": 0.07303018867969513, + "boolq": -0.2071463622544941, + "bigbench_language_identification": 0.18074805875553682 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/samples.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..38064e1d01244bd91b9c550f04e2b0471a1085d2 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the kingdom of France, and the capital of the kingdom of England" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the gold of the earth, and the gold of the earth is the gold of" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day will be Sunday. The day will be Sunday, and" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as that of cold, and the same as that of hot. The" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: the sun, the moon, the stars, the planets, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the color of the sky, and the color of the earth. The color of" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the sum of the sum of the sum of the sum of the sum of the" + } + ], + "unconditioned_samples": [ + "<|bos|>CHAPTER VS. CREV. A. It has already been mentioned that Roger North and the Anglo- Saxon Chronicle and the reigns of Henry and Edward III. tell us that Eden was situated on England's Apple within the government of Richard II., and it must now be premised that England brought over men from Normandy and Anjou, and maintained them there. The folly and swiftness of our pilgrims were its source.\n\nProvidence was no idler in being driven over this island by the rivers of Canterbury and Canterbury, and thence overland to Rome; nor in being conveyed to Jerusalem, to meet the friendlessness of an ignorant man", + "<|bos|>PREFACE.\n\nTHE portions of this work, which, without modification of time, would be impossible to be within the little compass of the compass, are so valuable, that it must be judged more as a class; and all men, of every class, are judged by the same standards of skill, observed by the same standard of practice.\n\nThe first part of the Cloaca Maxima, which was written in print, is too exuberant and exuberant, to allow of annotation. \"But the question came up whilst we were considering the books,\" he adds, \"of Ministers, And the gentlemen that were about me, joined in the", + "<|bos|>PREFACE.\n\nix emphatic word is given to such relief; and in the language of filial affection or maternal emotion or affection, the language he employs is never \"the language of all true repentance and deep sorrow, but a resort\" of sighs or complaints to a Father to whom all are alike reconciled, whose infinite attributes are majesty, loving-kindness, mercy, and justice.\n\nIf the incipient man was a material object or an object, he would \"stalk;\" by indolence, by vice, by attraction, he would throw himself into a stagnant pool, throw into it little basins and merely onions, drenching", + "<|bos|> Dreams and Rejoice over my Lute and Pledge I've Soon Began to\n\nWish never the pains of keep as Oblation all my Steps I took to Narrow the bounds I had to set;\n\nExtanced at this Stage my Branding spowl yet\n\nSeemed it a rather\n\nSpeech and gn\n\nA considerable Quantity Flapping in my 80 whiteness forbid the Onely False ity did afford me;\n\nAnd whoever Fidelity's Man bring in all South about 100\n\nEtched upon Conscience an Evoluble Sport Gold upon there a Tremb Billail he Built I had little faith till", + "<|bos|>Drops, with apples nourishing them and lilies suiting them. It is a custom with corn-growers to mow loaves. It is thus with the rusted corn, which we call wheat because it is hard. If a may be properly branched out into layers and used in raising and other purposes, it is sometimes described as branched out simply for bed, and germinated like a cap or knife, into one or other of the ultimate layers. It sometimes happens that pipes are left, which being laid on the ground and decayed or damped in time as they decompose them, remain", + "<|bos|>VE ZABE. [Heir of the Superinto- FOW is a gener- Founding of the Morality | Sociessie, by the Origin of Lan- Turks Previous to his elevation strength of Language, very spe- ties, been endowed with this essential business, ago there are only three distinct untitutional faculties. But the reconstruction of modern sciences in the Shinto variant is a very daring one, and treats the sublimity of the Fourteenth century as such. Storia in three skins parapped off byila- Strumeton, Nev., drawn him away and merrie (Wat v. Ortsboth", + "<|bos|>PREFACE Ruth and Robin Hood suggested the latter an asylum in the outhouses at Oakum Orbeek, where the late Charles Lee, the publisher of 1 A little instalment of this and other historical works is printed at the end of 1872.\n\nPREFACE.\n\nto secure a place for retirement should he be ever called on to join the rebels. This was left the further task of keeping all the patent offices in the selection of Warwick the publisher of aegicide Liber Quintale. \"Master Pashan and the vae variarent,\" added Francis Gorton after the interview above to Pashan's title,", + "<|bos|>opers.\n\n CarthABELLA LUBI.\n\n221 however, still perseveres (1251 n. d.), and succeeded in breaking down the walls and overwhelming the rest with stones. 66 It appears,\" says Lubio (d.) I, \"that the free city is secure in its centre; and that I, who have interfered on behalf of my people at all times during my rule, can easily turn the safety of Smyrna when made famous by the massacre of the Amir and the surrender of the city on the appointed day.\" This is somewhat like about Cortez: \"Franciscus qui desitunt\n\n" + ] +} \ No newline at end of file diff --git a/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/val_bpb.json b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..ebea9b2c25c960bc6629627d0f7560538bddcc75 --- /dev/null +++ b/experiments/clean1930s-d12-r11.25-muoneq-muonplus/evals/val_bpb.json @@ -0,0 +1,12 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val": 1.0583351355174524 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file