| { |
| "catalog_name": "CogNet-1B Training Datasets", |
| "version": "1.0", |
| "date": "2026-06-13", |
| "tokenizer": "CharTokenizer v3 (136 vocab: ASCII + French accents)", |
| "domains": { |
| "CODE": { |
| "datasets": [ |
| { |
| "name": "the_stack_smol", |
| "hf_id": "bigcode/the-stack-smol", |
| "size": "~2.6 GB, 30 languages x 10K files", |
| "description": "Multi-language source code (Python, JS, C, C++, Java, Rust, Go, TypeScript). Permissively licensed subset of The Stack.", |
| "priority": 1, |
| "notes": "Stream by language. Filter to target languages. Use data_dir='data/{lang}'." |
| }, |
| { |
| "name": "codeparrot_clean", |
| "hf_id": "codeparrot/codeparrot-clean", |
| "size": "~50 GB, 5.36M Python files", |
| "description": "Deduplicated, quality-filtered Python code. Removes auto-generated code and boilerplate.", |
| "priority": 2, |
| "notes": "Best Python-only dataset. Extract 'content' field. Mostly ASCII already." |
| }, |
| { |
| "name": "code_alpaca", |
| "hf_id": "sahil2801/CodeAlpaca-20k", |
| "size": "~20K instruction-code pairs", |
| "description": "Multi-language code generation instructions in Alpaca format (instruction/input/output).", |
| "priority": 3, |
| "notes": "Flatten instruction+input+output with ### delimiters. Small but high quality." |
| }, |
| { |
| "name": "codesearchnet", |
| "hf_id": "code_search_net", |
| "size": "~2M functions across 6 languages", |
| "description": "Function definitions paired with docstrings/comments. Python, JavaScript, Java, Go, Ruby, PHP.", |
| "priority": 4, |
| "notes": "Extract func_code_string + func_documentation_string. Code+docs co-occurrence." |
| }, |
| { |
| "name": "python_code_instructions", |
| "hf_id": "iamtarun/python_code_instructions_18k_alpaca", |
| "size": "~18K Python instruction pairs", |
| "description": "Python-specific code generation instructions in Alpaca format.", |
| "priority": 5, |
| "notes": "Same format as CodeAlpaca. Python-only, mostly ASCII." |
| } |
| ], |
| "total_estimated_tokens": "2B-10B (depending on subset size)" |
| }, |
| "MATH": { |
| "datasets": [ |
| { |
| "name": "mathpile", |
| "hf_id": "zwhe99/mathpile-text", |
| "size": "~9.5B tokens, 100K+ documents", |
| "description": "Large-scale math pretraining corpus: textbooks, arXiv, Wikipedia math, ProofWiki, StackExchange.", |
| "priority": 1, |
| "notes": "PRETRAINING corpus (not Q&A) — ideal for char-level. Contains LaTeX (ASCII-friendly!)." |
| }, |
| { |
| "name": "openmath_instruct1", |
| "hf_id": "nvidia/OpenMathInstruct-1", |
| "size": "1.8M problem-solution pairs, ~8.94 GB", |
| "description": "Math instruction tuning with step-by-step solutions. Generated from GSM8K+MATH using Mixtral.", |
| "priority": 2, |
| "notes": "Extract problem + generated_solution. May contain LaTeX." |
| }, |
| { |
| "name": "metamath_qa", |
| "hf_id": "meta-math/MetaMathQA", |
| "size": "395K augmented math Q&A pairs, ~396 MB", |
| "description": "Bootstrapped math Q&A with rephrased questions. Covers GSM8K and MATH problems.", |
| "priority": 3, |
| "notes": "Extract query + response. Contains LaTeX." |
| }, |
| { |
| "name": "gsm8k", |
| "hf_id": "openai/gsm8k", |
| "size": "~8.5K grade-school math problems", |
| "description": "Classic grade-school math word problems with step-by-step solutions.", |
| "priority": 4, |
| "notes": "Small but high quality. Answers contain #### separators. Good for basic arithmetic." |
| }, |
| { |
| "name": "hendrycks_math", |
| "hf_id": "EleutherAI/hendrycks_math", |
| "size": "12.5K competition-level problems (7 subjects)", |
| "description": "Competition math (AMC/AIME). Algebra, Counting, Geometry, Number Theory, etc. Heavy LaTeX.", |
| "priority": 5, |
| "notes": "LaTeX is pure ASCII — natively compatible! Model will learn LaTeX syntax. Excellent for advanced math." |
| } |
| ], |
| "total_estimated_tokens": "5B-15B (depending on subset size)" |
| }, |
| "SYNTAX": { |
| "datasets": [ |
| { |
| "name": "wikitext103", |
| "hf_id": "Salesforce/wikitext", |
| "config": "wikitext-103-raw-v1", |
| "size": "~103M tokens, ~516 MB", |
| "description": "Wikipedia Good/Featured articles. Rich syntactic structures. Raw variant preserves characters.", |
| "priority": 1, |
| "notes": "USE THE RAW VARIANT. Non-raw replaces rare chars with <unk> which is useless for char-level." |
| }, |
| { |
| "name": "c4_subset", |
| "hf_id": "allenai/c4", |
| "config": "en", |
| "size": "~156B tokens (we'll take a ~5GB subset)", |
| "description": "Massive clean web text. Already cleaned/deduplicated. Natural syntax, grammar, diverse topics.", |
| "priority": 2, |
| "notes": "STREAMING MANDATORY. We'll cap at ~100K docs (~5GB). Best for broad syntax exposure." |
| }, |
| { |
| "name": "ptb", |
| "hf_id": "ptb_text_only", |
| "size": "~42K training sentences, ~1M words", |
| "description": "Gold-standard syntactically parsed English text (Wall Street Journal).", |
| "priority": 3, |
| "notes": "Small but high quality for syntax learning. Use raw text variant." |
| }, |
| { |
| "name": "universal_deps", |
| "hf_id": "universal-dependencies/universal_dependencies", |
| "size": "200+ treebanks in 150+ languages", |
| "description": "Dependency parses with POS tags, lemmas, morphological features. CoNLL-U format.", |
| "priority": 4, |
| "notes": "We use en_gum, en_ewt, fr_gsd, fr_sequoia. French treebanks provide accent exposure. CoNLL-U is pure ASCII." |
| } |
| ], |
| "total_estimated_tokens": "500M-2B (depending on C4 subset size)" |
| }, |
| "GENERAL": { |
| "datasets": [ |
| { |
| "name": "pile_subset", |
| "hf_id": "EleutherAI/pile", |
| "size": "825 GiB total, ~300B tokens (we'll take ~10GB subset)", |
| "description": "All-in-one: GitHub, arXiv, StackExchange, Wikipedia, books. Contains code, math, and prose.", |
| "priority": 1, |
| "notes": "We'll cap at ~200K docs. Excellent mixed-domain data. Some copyright concerns — Common Pile is an alternative." |
| } |
| ], |
| "total_estimated_tokens": "1B-5B (subset)" |
| } |
| }, |
| "total_estimated_tokens": "8B-32B (depending on storage budget and filtering)", |
| "storage_budget_note": "50GB default budget. Adjust with --max_gb flag.", |
| "preprocessing_strategy": { |
| "unicode_handling": "Map common Unicode to ASCII equivalents (smart quotes, em-dashes, math symbols)", |
| "latex_strategy": "Keep LaTeX as-is (it's pure ASCII). Model learns LaTeX natively.", |
| "french_accents": "Preserved via 136-char vocab. UD French treebanks + C4 French subset provide accent exposure.", |
| "separator": "Double newline (\\n\\n) between documents/samples", |
| "encoding": "Token IDs saved as int16 tensors (.pt files), then merged into train/val splits" |
| }, |
| "how_to_run": { |
| "full_run": "python cognet_data_prep.py --output_dir /root/CogNet/data_1b --max_gb 50", |
| "quick_test": "python cognet_data_prep.py --output_dir /root/CogNet/data_1b --max_gb 5 --dry_run", |
| "only_code": "python cognet_data_prep.py --only the_stack_smol code_alpaca", |
| "only_math": "python cognet_data_prep.py --only mathpile gsm8k hendrycks_math", |
| "with_custom_code": "python cognet_data_prep.py --custom_code_dir /path/to/code/files", |
| "skip_merge": "python cognet_data_prep.py --skip_merge" |
| } |
| } |
|
|
