File size: 1,549 Bytes
8662ab2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | {
"max_bytes": 200000,
"max_rows_per_source": 5000000,
"min_chars": 200,
"name": "sml-v2-educational-prose-v1",
"seed": 7337,
"shuffle_buffer": 10000,
"sources": [
{
"config": "fineweb-edu-dedup",
"label": "web",
"license": "ODC-BY; preserve underlying source notices",
"repo": "HuggingFaceTB/smollm-corpus",
"revision": "3ba9d605774198c5868892d7a8deda78031a781f",
"split": "train",
"text_field": "text",
"weight": 55
},
{
"label": "dclm",
"license": "CC-BY-4.0",
"min_score": {
"field": "edu_int_score",
"value": 3
},
"repo": "HuggingFaceTB/dclm-edu",
"revision": "dbad8ad71224482740cd9c9d353591adbf62fe04",
"split": "train",
"text_field": "text",
"weight": 25
},
{
"config": "en",
"label": "wiki",
"license": "CC-BY-SA-4.0 / GFDL",
"repo": "HuggingFaceFW/finewiki",
"revision": "8bd13e72e6a002407649b3e898535f42ceb1aeb9",
"split": "train",
"text_field": "text",
"weight": 10
},
{
"config": "cosmopedia-v2",
"format_contains": [
"textbook",
"tutorial",
"blog",
"educational"
],
"label": "cosmopedia",
"license": "ODC-BY; synthetic content; preserve provenance",
"repo": "HuggingFaceTB/smollm-corpus",
"revision": "3ba9d605774198c5868892d7a8deda78031a781f",
"split": "train",
"text_field": "text",
"weight": 10
}
],
"split_buckets": 10000
}
|