| { |
| "tokenizer_type": "ByteLevel BPE", |
| "vocab_size": 32000, |
| "target_texts": 700000, |
| "min_chars": 80, |
| "max_chars": 12000, |
| "datasets": [ |
| { |
| "dataset": "allenai/c4", |
| "config": "en", |
| "split": "train", |
| "column": "text", |
| "share": 0.5, |
| "target_texts": 350000 |
| }, |
| { |
| "dataset": "HuggingFaceFW/fineweb-edu", |
| "config": null, |
| "split": "train", |
| "column": "text", |
| "share": 0.2, |
| "target_texts": 140000 |
| }, |
| { |
| "dataset": "wikimedia/wikipedia", |
| "config": "20231101.en", |
| "split": "train", |
| "column": "text", |
| "share": 0.1, |
| "target_texts": 70000 |
| }, |
| { |
| "dataset": "codeparrot/codeparrot-clean", |
| "config": null, |
| "split": "train", |
| "column": "content", |
| "share": 0.1, |
| "target_texts": 70000 |
| }, |
| { |
| "dataset": "allenai/c4", |
| "config": "es", |
| "split": "train", |
| "column": "text", |
| "share": 0.1, |
| "target_texts": 70000 |
| } |
| ], |
| "elapsed_seconds": 209.82311129570007, |
| "elapsed_minutes": 3.4970518549283347, |
| "output_dir": "/out/tokenizer-bpe-32k" |
| } |
