{ "catalog_name": "CogNet-1B Training Datasets", "version": "1.0", "date": "2026-06-13", "tokenizer": "CharTokenizer v3 (136 vocab: ASCII + French accents)", "domains": { "CODE": { "datasets": [ { "name": "the_stack_smol", "hf_id": "bigcode/the-stack-smol", "size": "~2.6 GB, 30 languages x 10K files", "description": "Multi-language source code (Python, JS, C, C++, Java, Rust, Go, TypeScript). Permissively licensed subset of The Stack.", "priority": 1, "notes": "Stream by language. Filter to target languages. Use data_dir='data/{lang}'." }, { "name": "codeparrot_clean", "hf_id": "codeparrot/codeparrot-clean", "size": "~50 GB, 5.36M Python files", "description": "Deduplicated, quality-filtered Python code. Removes auto-generated code and boilerplate.", "priority": 2, "notes": "Best Python-only dataset. Extract 'content' field. Mostly ASCII already." }, { "name": "code_alpaca", "hf_id": "sahil2801/CodeAlpaca-20k", "size": "~20K instruction-code pairs", "description": "Multi-language code generation instructions in Alpaca format (instruction/input/output).", "priority": 3, "notes": "Flatten instruction+input+output with ### delimiters. Small but high quality." }, { "name": "codesearchnet", "hf_id": "code_search_net", "size": "~2M functions across 6 languages", "description": "Function definitions paired with docstrings/comments. Python, JavaScript, Java, Go, Ruby, PHP.", "priority": 4, "notes": "Extract func_code_string + func_documentation_string. Code+docs co-occurrence." }, { "name": "python_code_instructions", "hf_id": "iamtarun/python_code_instructions_18k_alpaca", "size": "~18K Python instruction pairs", "description": "Python-specific code generation instructions in Alpaca format.", "priority": 5, "notes": "Same format as CodeAlpaca. Python-only, mostly ASCII." } ], "total_estimated_tokens": "2B-10B (depending on subset size)" }, "MATH": { "datasets": [ { "name": "mathpile", "hf_id": "zwhe99/mathpile-text", "size": "~9.5B tokens, 100K+ documents", "description": "Large-scale math pretraining corpus: textbooks, arXiv, Wikipedia math, ProofWiki, StackExchange.", "priority": 1, "notes": "PRETRAINING corpus (not Q&A) — ideal for char-level. Contains LaTeX (ASCII-friendly!)." }, { "name": "openmath_instruct1", "hf_id": "nvidia/OpenMathInstruct-1", "size": "1.8M problem-solution pairs, ~8.94 GB", "description": "Math instruction tuning with step-by-step solutions. Generated from GSM8K+MATH using Mixtral.", "priority": 2, "notes": "Extract problem + generated_solution. May contain LaTeX." }, { "name": "metamath_qa", "hf_id": "meta-math/MetaMathQA", "size": "395K augmented math Q&A pairs, ~396 MB", "description": "Bootstrapped math Q&A with rephrased questions. Covers GSM8K and MATH problems.", "priority": 3, "notes": "Extract query + response. Contains LaTeX." }, { "name": "gsm8k", "hf_id": "openai/gsm8k", "size": "~8.5K grade-school math problems", "description": "Classic grade-school math word problems with step-by-step solutions.", "priority": 4, "notes": "Small but high quality. Answers contain #### separators. Good for basic arithmetic." }, { "name": "hendrycks_math", "hf_id": "EleutherAI/hendrycks_math", "size": "12.5K competition-level problems (7 subjects)", "description": "Competition math (AMC/AIME). Algebra, Counting, Geometry, Number Theory, etc. Heavy LaTeX.", "priority": 5, "notes": "LaTeX is pure ASCII — natively compatible! Model will learn LaTeX syntax. Excellent for advanced math." } ], "total_estimated_tokens": "5B-15B (depending on subset size)" }, "SYNTAX": { "datasets": [ { "name": "wikitext103", "hf_id": "Salesforce/wikitext", "config": "wikitext-103-raw-v1", "size": "~103M tokens, ~516 MB", "description": "Wikipedia Good/Featured articles. Rich syntactic structures. Raw variant preserves characters.", "priority": 1, "notes": "USE THE RAW VARIANT. Non-raw replaces rare chars with which is useless for char-level." }, { "name": "c4_subset", "hf_id": "allenai/c4", "config": "en", "size": "~156B tokens (we'll take a ~5GB subset)", "description": "Massive clean web text. Already cleaned/deduplicated. Natural syntax, grammar, diverse topics.", "priority": 2, "notes": "STREAMING MANDATORY. We'll cap at ~100K docs (~5GB). Best for broad syntax exposure." }, { "name": "ptb", "hf_id": "ptb_text_only", "size": "~42K training sentences, ~1M words", "description": "Gold-standard syntactically parsed English text (Wall Street Journal).", "priority": 3, "notes": "Small but high quality for syntax learning. Use raw text variant." }, { "name": "universal_deps", "hf_id": "universal-dependencies/universal_dependencies", "size": "200+ treebanks in 150+ languages", "description": "Dependency parses with POS tags, lemmas, morphological features. CoNLL-U format.", "priority": 4, "notes": "We use en_gum, en_ewt, fr_gsd, fr_sequoia. French treebanks provide accent exposure. CoNLL-U is pure ASCII." } ], "total_estimated_tokens": "500M-2B (depending on C4 subset size)" }, "GENERAL": { "datasets": [ { "name": "pile_subset", "hf_id": "EleutherAI/pile", "size": "825 GiB total, ~300B tokens (we'll take ~10GB subset)", "description": "All-in-one: GitHub, arXiv, StackExchange, Wikipedia, books. Contains code, math, and prose.", "priority": 1, "notes": "We'll cap at ~200K docs. Excellent mixed-domain data. Some copyright concerns — Common Pile is an alternative." } ], "total_estimated_tokens": "1B-5B (subset)" } }, "total_estimated_tokens": "8B-32B (depending on storage budget and filtering)", "storage_budget_note": "50GB default budget. Adjust with --max_gb flag.", "preprocessing_strategy": { "unicode_handling": "Map common Unicode to ASCII equivalents (smart quotes, em-dashes, math symbols)", "latex_strategy": "Keep LaTeX as-is (it's pure ASCII). Model learns LaTeX natively.", "french_accents": "Preserved via 136-char vocab. UD French treebanks + C4 French subset provide accent exposure.", "separator": "Double newline (\\n\\n) between documents/samples", "encoding": "Token IDs saved as int16 tensors (.pt files), then merged into train/val splits" }, "how_to_run": { "full_run": "python cognet_data_prep.py --output_dir /root/CogNet/data_1b --max_gb 50", "quick_test": "python cognet_data_prep.py --output_dir /root/CogNet/data_1b --max_gb 5 --dry_run", "only_code": "python cognet_data_prep.py --only the_stack_smol code_alpaca", "only_math": "python cognet_data_prep.py --only mathpile gsm8k hendrycks_math", "with_custom_code": "python cognet_data_prep.py --custom_code_dir /path/to/code/files", "skip_merge": "python cognet_data_prep.py --skip_merge" } }