Add cognet_datasets_catalog.json

data/cognet_datasets_catalog.json

@@ -0,0 +1,167 @@
+{
+  "catalog_name": "CogNet-1B Training Datasets",
+  "version": "1.0",
+  "date": "2026-06-13",
+  "tokenizer": "CharTokenizer v3 (136 vocab: ASCII + French accents)",
+  "domains": {
+    "CODE": {
+      "datasets": [
+        {
+          "name": "the_stack_smol",
+          "hf_id": "bigcode/the-stack-smol",
+          "size": "~2.6 GB, 30 languages x 10K files",
+          "description": "Multi-language source code (Python, JS, C, C++, Java, Rust, Go, TypeScript). Permissively licensed subset of The Stack.",
+          "priority": 1,
+          "notes": "Stream by language. Filter to target languages. Use data_dir='data/{lang}'."
+        },
+        {
+          "name": "codeparrot_clean",
+          "hf_id": "codeparrot/codeparrot-clean",
+          "size": "~50 GB, 5.36M Python files",
+          "description": "Deduplicated, quality-filtered Python code. Removes auto-generated code and boilerplate.",
+          "priority": 2,
+          "notes": "Best Python-only dataset. Extract 'content' field. Mostly ASCII already."
+        },
+        {
+          "name": "code_alpaca",
+          "hf_id": "sahil2801/CodeAlpaca-20k",
+          "size": "~20K instruction-code pairs",
+          "description": "Multi-language code generation instructions in Alpaca format (instruction/input/output).",
+          "priority": 3,
+          "notes": "Flatten instruction+input+output with ### delimiters. Small but high quality."
+        },
+        {
+          "name": "codesearchnet",
+          "hf_id": "code_search_net",
+          "size": "~2M functions across 6 languages",
+          "description": "Function definitions paired with docstrings/comments. Python, JavaScript, Java, Go, Ruby, PHP.",
+          "priority": 4,
+          "notes": "Extract func_code_string + func_documentation_string. Code+docs co-occurrence."
+        },
+        {
+          "name": "python_code_instructions",
+          "hf_id": "iamtarun/python_code_instructions_18k_alpaca",
+          "size": "~18K Python instruction pairs",
+          "description": "Python-specific code generation instructions in Alpaca format.",
+          "priority": 5,
+          "notes": "Same format as CodeAlpaca. Python-only, mostly ASCII."
+        }
+      ],
+      "total_estimated_tokens": "2B-10B (depending on subset size)"
+    },
+    "MATH": {
+      "datasets": [
+        {
+          "name": "mathpile",
+          "hf_id": "zwhe99/mathpile-text",
+          "size": "~9.5B tokens, 100K+ documents",
+          "description": "Large-scale math pretraining corpus: textbooks, arXiv, Wikipedia math, ProofWiki, StackExchange.",
+          "priority": 1,
+          "notes": "PRETRAINING corpus (not Q&A) — ideal for char-level. Contains LaTeX (ASCII-friendly!)."
+        },
+        {
+          "name": "openmath_instruct1",
+          "hf_id": "nvidia/OpenMathInstruct-1",
+          "size": "1.8M problem-solution pairs, ~8.94 GB",
+          "description": "Math instruction tuning with step-by-step solutions. Generated from GSM8K+MATH using Mixtral.",
+          "priority": 2,
+          "notes": "Extract problem + generated_solution. May contain LaTeX."
+        },
+        {
+          "name": "metamath_qa",
+          "hf_id": "meta-math/MetaMathQA",
+          "size": "395K augmented math Q&A pairs, ~396 MB",
+          "description": "Bootstrapped math Q&A with rephrased questions. Covers GSM8K and MATH problems.",
+          "priority": 3,
+          "notes": "Extract query + response. Contains LaTeX."
+        },
+        {
+          "name": "gsm8k",
+          "hf_id": "openai/gsm8k",
+          "size": "~8.5K grade-school math problems",
+          "description": "Classic grade-school math word problems with step-by-step solutions.",
+          "priority": 4,
+          "notes": "Small but high quality. Answers contain #### separators. Good for basic arithmetic."
+        },
+        {
+          "name": "hendrycks_math",
+          "hf_id": "EleutherAI/hendrycks_math",
+          "size": "12.5K competition-level problems (7 subjects)",
+          "description": "Competition math (AMC/AIME). Algebra, Counting, Geometry, Number Theory, etc. Heavy LaTeX.",
+          "priority": 5,
+          "notes": "LaTeX is pure ASCII — natively compatible! Model will learn LaTeX syntax. Excellent for advanced math."
+        }
+      ],
+      "total_estimated_tokens": "5B-15B (depending on subset size)"
+    },
+    "SYNTAX": {
+      "datasets": [
+        {
+          "name": "wikitext103",
+          "hf_id": "Salesforce/wikitext",
+          "config": "wikitext-103-raw-v1",
+          "size": "~103M tokens, ~516 MB",
+          "description": "Wikipedia Good/Featured articles. Rich syntactic structures. Raw variant preserves characters.",
+          "priority": 1,
+          "notes": "USE THE RAW VARIANT. Non-raw replaces rare chars with <unk> which is useless for char-level."
+        },
+        {
+          "name": "c4_subset",
+          "hf_id": "allenai/c4",
+          "config": "en",
+          "size": "~156B tokens (we'll take a ~5GB subset)",
+          "description": "Massive clean web text. Already cleaned/deduplicated. Natural syntax, grammar, diverse topics.",
+          "priority": 2,
+          "notes": "STREAMING MANDATORY. We'll cap at ~100K docs (~5GB). Best for broad syntax exposure."
+        },
+        {
+          "name": "ptb",
+          "hf_id": "ptb_text_only",
+          "size": "~42K training sentences, ~1M words",
+          "description": "Gold-standard syntactically parsed English text (Wall Street Journal).",
+          "priority": 3,
+          "notes": "Small but high quality for syntax learning. Use raw text variant."
+        },
+        {
+          "name": "universal_deps",
+          "hf_id": "universal-dependencies/universal_dependencies",
+          "size": "200+ treebanks in 150+ languages",
+          "description": "Dependency parses with POS tags, lemmas, morphological features. CoNLL-U format.",
+          "priority": 4,
+          "notes": "We use en_gum, en_ewt, fr_gsd, fr_sequoia. French treebanks provide accent exposure. CoNLL-U is pure ASCII."
+        }
+      ],
+      "total_estimated_tokens": "500M-2B (depending on C4 subset size)"
+    },
+    "GENERAL": {
+      "datasets": [
+        {
+          "name": "pile_subset",
+          "hf_id": "EleutherAI/pile",
+          "size": "825 GiB total, ~300B tokens (we'll take ~10GB subset)",
+          "description": "All-in-one: GitHub, arXiv, StackExchange, Wikipedia, books. Contains code, math, and prose.",
+          "priority": 1,
+          "notes": "We'll cap at ~200K docs. Excellent mixed-domain data. Some copyright concerns — Common Pile is an alternative."
+        }
+      ],
+      "total_estimated_tokens": "1B-5B (subset)"
+    }
+  },
+  "total_estimated_tokens": "8B-32B (depending on storage budget and filtering)",
+  "storage_budget_note": "50GB default budget. Adjust with --max_gb flag.",
+  "preprocessing_strategy": {
+    "unicode_handling": "Map common Unicode to ASCII equivalents (smart quotes, em-dashes, math symbols)",
+    "latex_strategy": "Keep LaTeX as-is (it's pure ASCII). Model learns LaTeX natively.",
+    "french_accents": "Preserved via 136-char vocab. UD French treebanks + C4 French subset provide accent exposure.",
+    "separator": "Double newline (\\n\\n) between documents/samples",
+    "encoding": "Token IDs saved as int16 tensors (.pt files), then merged into train/val splits"
+  },
+  "how_to_run": {
+    "full_run": "python cognet_data_prep.py --output_dir /root/CogNet/data_1b --max_gb 50",
+    "quick_test": "python cognet_data_prep.py --output_dir /root/CogNet/data_1b --max_gb 5 --dry_run",
+    "only_code": "python cognet_data_prep.py --only the_stack_smol code_alpaca",
+    "only_math": "python cognet_data_prep.py --only mathpile gsm8k hendrycks_math",
+    "with_custom_code": "python cognet_data_prep.py --custom_code_dir /path/to/code/files",
+    "skip_merge": "python cognet_data_prep.py --skip_merge"
+  }
+}