configs/datasets.yaml

# Dataset configurations for ULTRATHINK training
# Users can easily switch between datasets by modifying these configs

# Small datasets (good for testing and development)
wikitext:
  name: "wikitext"
  subset: "wikitext-2-raw-v1"
  description: "Small Wikipedia dataset (~100MB)"
  max_length: 512
  streaming: false
  recommended_for: "development, testing, quick experiments"

wikitext_large:
  name: "wikitext" 
  subset: "wikitext-103-raw-v1"
  description: "Larger Wikipedia dataset (~500MB)"
  max_length: 1024
  streaming: false
  recommended_for: "medium-scale training"

# Medium datasets (good for serious training)
openwebtext:
  name: "openwebtext"
  subset: null
  description: "Open source recreation of WebText (~40GB)"
  max_length: 1024
  streaming: true
  recommended_for: "serious training, diverse web content"

bookcorpus:
  name: "bookcorpus"
  subset: null
  description: "Collection of over 11,000 books (~5GB)"
  max_length: 1024
  streaming: true
  recommended_for: "literary and narrative training"

# Large datasets (for production training)
pile:
  name: "EleutherAI/pile"
  subset: null
  description: "Large-scale curated dataset (~800GB)"
  max_length: 2048
  streaming: true
  recommended_for: "large-scale production training"

c4:
  name: "c4"
  subset: "en"
  description: "Colossal Clean Crawled Corpus (~750GB)"
  max_length: 512
  streaming: true
  recommended_for: "large-scale web data training"

# Specialized datasets
oscar:
  name: "oscar"
  subset: "unshuffled_deduplicated_en"
  description: "Multilingual web crawl (English subset)"
  max_length: 512
  streaming: true
  recommended_for: "web content, multilingual training"

# Custom dataset template
custom_example:
  name: "custom"
  local_path: "/path/to/your/dataset.json"
  file_type: "json"  # json, txt, csv, parquet
  text_column: "text"
  max_length: 512
  streaming: false
  description: "Template for custom datasets"
  recommended_for: "domain-specific training"

# Mixed dataset example
mixed_example:
  datasets:
    wikitext: 0.3
    openwebtext: 0.5
    bookcorpus: 0.2
  description: "Mix multiple datasets with weights"
  recommended_for: "balanced training across domains"

# Dataset recommendations by use case
recommendations:
  quick_test: "wikitext"
  development: "wikitext_large"
  research: "openwebtext"
  production: "pile"
  literary: "bookcorpus"
  web_content: "c4"
  multilingual: "oscar"