| { | |
| "format": "bananamind2_pico_tokenizer_v1", | |
| "stage": "tokenizer_complete", | |
| "vocab_size": 384, | |
| "model_max_length": 8192, | |
| "target_gib": 1.0, | |
| "sources": [ | |
| { | |
| "name": "fineweb_hq", | |
| "dataset_id": "epfml/FineWeb-HQ", | |
| "config": null, | |
| "text_field": "text", | |
| "byte_share": 0.75, | |
| "revision": "e58199cdd52438d94405df1a4d8630cc5f13bf84", | |
| "target_bytes": 805306368, | |
| "consumed_bytes": 805307468 | |
| }, | |
| { | |
| "name": "cosmopedia_v2", | |
| "dataset_id": "HuggingFaceTB/smollm-corpus", | |
| "config": "cosmopedia-v2", | |
| "text_field": "text", | |
| "byte_share": 0.25, | |
| "revision": "3ba9d605774198c5868892d7a8deda78031a781f", | |
| "target_bytes": 268435456, | |
| "consumed_bytes": 268438475 | |
| } | |
| ] | |
| } | |