| # Comprehensive data sources for Supernova - covering all subjects and fields of knowledge | |
| # This configuration ensures broad coverage across every domain of human knowledge | |
| sources: | |
| # Core Web Crawl Data (General Knowledge) | |
| - name: c4_en | |
| hf_path: c4 | |
| hf_name: en | |
| split: train | |
| text_field: text | |
| weight: 10 | |
| streaming: true | |
| - name: openwebtext | |
| hf_path: openwebtext | |
| hf_name: null | |
| split: train | |
| text_field: text | |
| weight: 8 | |
| streaming: true | |
| - name: the_pile | |
| hf_path: the_pile | |
| hf_name: all | |
| split: train | |
| text_field: text | |
| weight: 15 | |
| streaming: true | |
| # Encyclopedia & Reference (Structured Knowledge) | |
| - name: wikipedia_en | |
| hf_path: wikipedia | |
| hf_name: 20220301.en | |
| split: train | |
| text_field: text | |
| weight: 12 | |
| streaming: true | |
| # Literature & Humanities | |
| - name: bookcorpusopen | |
| hf_path: bookcorpusopen | |
| hf_name: null | |
| split: train | |
| text_field: text | |
| weight: 6 | |
| streaming: true | |
| - name: gutenberg_books | |
| hf_path: sedthh/gutenberg_english | |
| hf_name: null | |
| split: train | |
| text_field: text | |
| weight: 4 | |
| streaming: true | |
| # Academic & Scientific Papers | |
| - name: arxiv_papers | |
| hf_path: togethercomputer/RedPajama-Data-1T | |
| hf_name: arxiv | |
| split: train | |
| text_field: text | |
| weight: 8 | |
| streaming: true | |
| - name: pubmed_abstracts | |
| hf_path: togethercomputer/RedPajama-Data-1T | |
| hf_name: pubmed_abstracts | |
| split: train | |
| text_field: text | |
| weight: 6 | |
| streaming: true | |
| # Code & Technical Documentation | |
| - name: github_code | |
| hf_path: togethercomputer/RedPajama-Data-1T | |
| hf_name: github | |
| split: train | |
| text_field: text | |
| weight: 7 | |
| streaming: true | |
| - name: stack_exchange | |
| hf_path: togethercomputer/RedPajama-Data-1T | |
| hf_name: stackexchange | |
| split: train | |
| text_field: text | |
| weight: 5 | |
| streaming: true | |
| # Mathematics & Science Specific | |
| - name: math_dataset | |
| hf_path: competition_math | |
| hf_name: null | |
| split: train | |
| text_field: problem | |
| weight: 3 | |
| streaming: true | |
| - name: scientific_papers | |
| hf_path: allenai/s2orc | |
| hf_name: null | |
| split: train | |
| text_field: text | |
| weight: 6 | |
| streaming: true | |
| # News & Current Events (for general knowledge) | |
| - name: cc_news | |
| hf_path: togethercomputer/RedPajama-Data-1T | |
| hf_name: cc_news | |
| split: train | |
| text_field: text | |
| weight: 4 | |
| streaming: true | |
| # Educational Content | |
| - name: khan_academy | |
| hf_path: prasadsharaf/khan-academy-scrape | |
| hf_name: null | |
| split: train | |
| text_field: text | |
| weight: 3 | |
| streaming: true | |
| # Legal Documents (Law) | |
| - name: legal_pile | |
| hf_path: pile-of-law/pile-of-law | |
| hf_name: null | |
| split: train | |
| text_field: text | |
| weight: 2 | |
| streaming: true | |
| # Medical & Healthcare | |
| - name: medical_meadow | |
| hf_path: medalpaca/medical_meadow_medical_flashcards | |
| hf_name: null | |
| split: train | |
| text_field: output | |
| weight: 2 | |
| streaming: true | |
| # Philosophy & Ethics | |
| - name: philosophy_dataset | |
| hf_path: AiresPucrs/stanford-encyclopedia-philosophy | |
| hf_name: null | |
| split: train | |
| text_field: text | |
| weight: 2 | |
| streaming: true | |
| # Note: Some datasets might require authentication or have usage restrictions | |
| # Always review the license and terms of use for each dataset | |
| # Adjust weights based on your priorities and available compute resources | |
| # Higher weights = more representation in training | |
| # Coverage areas: | |
| # ✓ General Web Knowledge (C4, OpenWebText, The Pile) | |
| # ✓ Encyclopedic Knowledge (Wikipedia) | |
| # ✓ Literature & Arts (Books, Gutenberg) | |
| # ✓ Science & Research (ArXiv, PubMed, S2ORC) | |
| # ✓ Technology & Programming (GitHub, Stack Exchange) | |
| # ✓ Mathematics (Competition Math, Scientific Papers) | |
| # ✓ Current Events (News) | |
| # ✓ Education (Khan Academy) | |
| # ✓ Law (Pile of Law) | |
| # ✓ Medicine (Medical datasets) | |
| # ✓ Philosophy & Ethics | |
| # ✓ Engineering (through technical papers and code) | |
| # ✓ History (through Wikipedia and books) | |
| # ✓ Languages & Linguistics (through diverse text sources) | |
| # ✓ Business & Economics (through news and web content) |