{
  "max_bytes": 200000,
  "max_rows_per_source": 5000000,
  "min_chars": 200,
  "name": "sml-v2-educational-prose-v1",
  "seed": 7337,
  "shuffle_buffer": 10000,
  "sources": [
    {
      "config": "fineweb-edu-dedup",
      "label": "web",
      "license": "ODC-BY; preserve underlying source notices",
      "repo": "HuggingFaceTB/smollm-corpus",
      "revision": "3ba9d605774198c5868892d7a8deda78031a781f",
      "split": "train",
      "text_field": "text",
      "weight": 55
    },
    {
      "label": "dclm",
      "license": "CC-BY-4.0",
      "min_score": {
        "field": "edu_int_score",
        "value": 3
      },
      "repo": "HuggingFaceTB/dclm-edu",
      "revision": "dbad8ad71224482740cd9c9d353591adbf62fe04",
      "split": "train",
      "text_field": "text",
      "weight": 25
    },
    {
      "config": "en",
      "label": "wiki",
      "license": "CC-BY-SA-4.0 / GFDL",
      "repo": "HuggingFaceFW/finewiki",
      "revision": "8bd13e72e6a002407649b3e898535f42ceb1aeb9",
      "split": "train",
      "text_field": "text",
      "weight": 10
    },
    {
      "config": "cosmopedia-v2",
      "format_contains": [
        "textbook",
        "tutorial",
        "blog",
        "educational"
      ],
      "label": "cosmopedia",
      "license": "ODC-BY; synthetic content; preserve provenance",
      "repo": "HuggingFaceTB/smollm-corpus",
      "revision": "3ba9d605774198c5868892d7a8deda78031a781f",
      "split": "train",
      "text_field": "text",
      "weight": 10
    }
  ],
  "split_buckets": 10000
}