Upload Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1 step 160

experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1/checkpoints/meta_000160.json

@@ -0,0 +1,250 @@
+{
+  "step": 160,
+  "training_complete": true,
+  "val_bpb": 0.32778816957977247,
+  "total_batch_size": 2097152,
+  "model_config": {
+    "sequence_len": 4096,
+    "vocab_size": 32768,
+    "n_layer": 32,
+    "n_head": 16,
+    "n_kv_head": 16,
+    "n_embd": 2048,
+    "window_pattern": "SSSL"
+  },
+  "user_config": {
+    "run": "Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1",
+    "wandb_run_id": "8c553682",
+    "wandb_group": "think-d32",
+    "wandb_tags": "sft,modern,nanochat-default,data-matched,compute-controlled,d32",
+    "device_type": "",
+    "model_tag": null,
+    "model_step": null,
+    "base_checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/base_checkpoints",
+    "base_step": 9600,
+    "checkpoint_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1/checkpoints",
+    "tokenizer_dir": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/tokenizer",
+    "resume_from_step": null,
+    "experiment_id": "Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1",
+    "experiment_config": "/workspace/nanochat/experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1/config.json",
+    "parent_cumulative_flops": 2.3103503196928082e+20,
+    "tokenizer_fingerprint": "21e99d5cdeeaa660",
+    "git_commit_sha": "3c34775d260a016a79ab46ea76f8c8e48458fd68",
+    "load_optimizer": 0,
+    "num_iterations": -1,
+    "num_epochs": 1,
+    "target_examples_per_step": 0,
+    "max_seq_len": null,
+    "device_batch_size": 2,
+    "total_batch_size": null,
+    "embedding_lr": null,
+    "unembedding_lr": null,
+    "matrix_lr": null,
+    "init_lr_frac": 0.8,
+    "warmup_ratio": 0.03,
+    "warmdown_ratio": 0.5,
+    "final_lr_frac": 0.0,
+    "eval_every": -1,
+    "eval_tokens": 20971520,
+    "chatcore_every": -1,
+    "chatcore_max_cat": -1,
+    "chatcore_max_sample": 32,
+    "save_every": -1,
+    "recipe": "nanochat-default",
+    "curriculum_config": "",
+    "pre1930_epochs": 5,
+    "mmlu_epochs": 3,
+    "gsm8k_epochs": 4,
+    "max_train_presentations": 652950,
+    "authentic_epochs": 0,
+    "knowledge_qa_epochs": 0,
+    "multiturn_qa_epochs": 0,
+    "reasoning_qa_epochs": 0,
+    "stem_reasoning_epochs": 0,
+    "narrative_grounded_epochs": 0,
+    "narrative_fiction_epochs": 0,
+    "opinion_qa_epochs": 0,
+    "how_to_qa_epochs": 0,
+    "verse_qa_epochs": 0,
+    "composition_qa_epochs": 0,
+    "resolved_experiment_config": {
+      "schema_version": 1,
+      "stage": "sft",
+      "experiment_suffix": "nanochat-default-datamatch-v1",
+      "data": {
+        "recipe": "nanochat-default",
+        "mmlu_epochs": 3,
+        "gsm8k_epochs": 4,
+        "max_train_presentations": 652950
+      },
+      "training": {
+        "num_iterations": -1,
+        "load_optimizer": 0,
+        "device_batch_size": 2,
+        "warmup_ratio": 0.03,
+        "warmdown_ratio": 0.5,
+        "init_lr_frac": 0.8,
+        "final_lr_frac": 0.0,
+        "eval_every": -1,
+        "chatcore_every": -1,
+        "save_every": -1
+      },
+      "comparison": {
+        "reference_sft": "pre1930-curriculum-c3-robust-v2",
+        "reference_train_presentations": 652950,
+        "reference_optimizer_steps": 42,
+        "reference_stage_training_flops": 1.0107782648656036e+18,
+        "matching_priority": "row_presentations",
+        "note": "Realized compute is measured because conversation lengths differ."
+      },
+      "artifacts": {
+        "repo": "jbduran/think.nano"
+      },
+      "wandb": {
+        "enabled": true,
+        "entity": "jbduran-thinkingmachinesncsu",
+        "project": "think.nano",
+        "group": "think-d32",
+        "tags": [
+          "sft",
+          "modern",
+          "nanochat-default",
+          "data-matched",
+          "compute-controlled",
+          "d32"
+        ]
+      },
+      "config_fingerprint": "6359be7150ece075",
+      "artifact_path": "experiments/Think.Unbounded-d32-v2mix-cont/sft/Think.Unbounded-d32-v2mix-cont-nanochat-default-datamatch-v1"
+    },
+    "stage": "sft",
+    "base_experiment_id": null,
+    "parent_experiment_id": null,
+    "parent_checkpoint_step": null,
+    "config_fingerprint": "6359be7150ece075"
+  },
+  "sft_mixture_summary": {
+    "schema_version": 1,
+    "recipe": "nanochat-default",
+    "selection": {
+      "algorithm": "TaskMixture(seed=42) deterministic global shuffle then prefix",
+      "selected_presentations": 652950,
+      "full_mixture_presentations": 1071759,
+      "selection_sha256": "975f61958382acd3a3cd9b01c2f3204cff8982908365503ac8972e29d38bfec2"
+    },
+    "sources": {
+      "smoltalk": {
+        "kind": "huggingface",
+        "repo": "HuggingFaceTB/smol-smoltalk",
+        "split": "train",
+        "dataset_fingerprint": "7776c4bbcd5d614f",
+        "configured_repetitions": 1,
+        "available_rows": 460341,
+        "resolved_revision": "f73fe857d519ff6ac5af2ea67c4d3834da7b8bcc",
+        "full_mixture_presentations": 460341,
+        "selected_presentations": 280414,
+        "selected_presentations_by_replica": [
+          280414
+        ],
+        "selected_distinct_rows": 280414,
+        "selected_fraction": 0.4294570794088368
+      },
+      "identity": {
+        "kind": "jsonl",
+        "url": "https://karpathy-public.s3.us-west-2.amazonaws.com/identity_conversations.jsonl",
+        "sha256": "85a6a96342b88addeb1b8267933c47a6e3ceb1dfc07b305b805fd7ff8106878e",
+        "configured_repetitions": 2,
+        "available_rows": 1000,
+        "full_mixture_presentations": 2000,
+        "selected_presentations": 1241,
+        "selected_presentations_by_replica": [
+          630,
+          611
+        ],
+        "selected_distinct_rows": 874,
+        "selected_fraction": 0.0019006049467799986
+      },
+      "mmlu": {
+        "kind": "huggingface",
+        "repo": "cais/mmlu",
+        "subset": "all",
+        "split": "auxiliary_train",
+        "dataset_fingerprint": "5e98d840a8432dc6",
+        "configured_repetitions": 3,
+        "available_rows": 99842,
+        "resolved_revision": "c30699e8356da336a370243923dbaf21066bb9fe",
+        "full_mixture_presentations": 299526,
+        "selected_presentations": 182570,
+        "selected_presentations_by_replica": [
+          60835,
+          60932,
+          60803
+        ],
+        "selected_distinct_rows": 93775,
+        "selected_fraction": 0.2796079332261276
+      },
+      "gsm8k": {
+        "kind": "huggingface",
+        "repo": "openai/gsm8k",
+        "subset": "main",
+        "split": "train",
+        "dataset_fingerprint": "8b93a690dcf7094e",
+        "configured_repetitions": 4,
+        "available_rows": 7473,
+        "resolved_revision": "740312add88f781978c0658806c59bc2815b9866",
+        "full_mixture_presentations": 29892,
+        "selected_presentations": 18182,
+        "selected_presentations_by_replica": [
+          4516,
+          4549,
+          4540,
+          4577
+        ],
+        "selected_distinct_rows": 7288,
+        "selected_fraction": 0.02784593000995482
+      },
+      "simple_spelling": {
+        "kind": "deterministic_generated",
+        "generator": "tasks.spellingbee.SimpleSpelling",
+        "word_list_url": "https://raw.githubusercontent.com/dwyl/english-words/refs/heads/master/words_alpha.txt",
+        "word_list_sha256": "643c7f71d5caee9806be56f5ed83ba36e2246a06a04bcf4aaa0576790bb90d66",
+        "configured_repetitions": 1,
+        "available_rows": 200000,
+        "full_mixture_presentations": 200000,
+        "selected_presentations": 121661,
+        "selected_presentations_by_replica": [
+          121661
+        ],
+        "selected_distinct_rows": 121661,
+        "selected_fraction": 0.18632513975036374
+      },
+      "spelling_bee": {
+        "kind": "deterministic_generated",
+        "generator": "tasks.spellingbee.SpellingBee",
+        "word_list_url": "https://raw.githubusercontent.com/dwyl/english-words/refs/heads/master/words_alpha.txt",
+        "word_list_sha256": "643c7f71d5caee9806be56f5ed83ba36e2246a06a04bcf4aaa0576790bb90d66",
+        "configured_repetitions": 1,
+        "available_rows": 80000,
+        "full_mixture_presentations": 80000,
+        "selected_presentations": 48882,
+        "selected_presentations_by_replica": [
+          48882
+        ],
+        "selected_distinct_rows": 48882,
+        "selected_fraction": 0.07486331265793705
+      }
+    }
+  },
+  "loop_state": {
+    "step": 160,
+    "total_training_time": 6951.126858472824,
+    "min_val_bpb": 0.32778816957977247,
+    "smooth_train_loss": 0.8890764356615257,
+    "mfu": 52.42433466002523,
+    "tok_per_sec": 45180,
+    "stage_training_flops": 3.8505838661546803e+18,
+    "stage_training_tokens": 335544320,
+    "inherited_parent_flops": 2.3103503196928082e+20,
+    "cumulative_pipeline_training_flops": 2.348856158354355e+20
+  }
+}