| { | |
| "engine": "tokforge", | |
| "updated_at": "2026-08-19T20:55:04Z", | |
| "bucket": "domofon/datasets", | |
| "corpora": [ | |
| { | |
| "corpus": "cosmo", | |
| "labels": [ | |
| "pretrain", | |
| "fresh5b", | |
| "smollm-corpus", | |
| "topup" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "5b", | |
| "prefix": "cosmo5b-00", | |
| "shards": 9, | |
| "tokens": 1084355044, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "cosmo5b-01", | |
| "shards": 9, | |
| "tokens": 1084435214, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "cosmo5b-02", | |
| "shards": 9, | |
| "tokens": 1084565120, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "cosmo5b-03", | |
| "shards": 9, | |
| "tokens": 1084038125, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "cosmo5b-04", | |
| "shards": 3, | |
| "tokens": 271321339, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "cosmo5b-t0", | |
| "shards": 5, | |
| "tokens": 542268102, | |
| "tokens_available": 542268102, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "cosmo5b-t1", | |
| "shards": 5, | |
| "tokens": 542189138, | |
| "tokens_available": 542189138, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "cosmo2", | |
| "labels": [ | |
| "pretrain", | |
| "synthetic", | |
| "phase6", | |
| "resume-kill9" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "v6", | |
| "prefix": "cosmo6b-t0", | |
| "shards": 9, | |
| "tokens": 1083986721, | |
| "tokens_available": 1083986721, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v10", | |
| "prefix": "tokforge-test-v10", | |
| "shards": 26, | |
| "tokens": 5211657, | |
| "tokens_available": 5211657, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v11", | |
| "prefix": "tokforge-test-v11", | |
| "shards": 20, | |
| "tokens": 4006435, | |
| "tokens_available": 4006435, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v8", | |
| "prefix": "tokforge-test-v8", | |
| "shards": 15, | |
| "tokens": 3005929, | |
| "tokens_available": 3005929, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v9", | |
| "prefix": "tokforge-test-v9", | |
| "shards": 20, | |
| "tokens": 4006353, | |
| "tokens_available": 4006353, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "dclm", | |
| "labels": [ | |
| "pretrain", | |
| "fresh5b", | |
| "dclm-baseline-1.0", | |
| "topup" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "5b", | |
| "prefix": "dclm5b-00", | |
| "shards": 9, | |
| "tokens": 1122811061, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "dclm5b-01", | |
| "shards": 9, | |
| "tokens": 1026512352, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "dclm5b-02", | |
| "shards": 10, | |
| "tokens": 1143913738, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "dclm5b-03", | |
| "shards": 10, | |
| "tokens": 1198462259, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "dclm5b-04", | |
| "shards": 1, | |
| "tokens": 118922458, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "dclm5b-t0", | |
| "shards": 2, | |
| "tokens": 221399039, | |
| "tokens_available": 221399039, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "dclm5b-t1", | |
| "shards": 3, | |
| "tokens": 326670945, | |
| "tokens_available": 326670945, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "dolma3", | |
| "labels": [ | |
| "pretrain", | |
| "en", | |
| "production", | |
| "topup", | |
| "reserve", | |
| "mix10b", | |
| "smoke" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "v1", | |
| "prefix": "dolma3-cc-v1", | |
| "shards": 34, | |
| "tokens": 4142408617, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v1b", | |
| "prefix": "dolma3-cc-v1b", | |
| "shards": 1, | |
| "tokens": 0, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v1c", | |
| "prefix": "dolma3-cc-v1c", | |
| "shards": 8, | |
| "tokens": 880607554, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v2", | |
| "prefix": "dolma3-cc-v2", | |
| "shards": 70, | |
| "tokens": 8630697760, | |
| "tokens_available": 130646054, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-01", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 49239555, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-02", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 49239555, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-03", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 49239555, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-04", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 49239555, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-05", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-06", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-07", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-08", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-09", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-10", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-11", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-12", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-13", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-14", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-15", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-16", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-17", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-18", | |
| "shards": 4, | |
| "tokens": 424240467, | |
| "tokens_available": 424240467, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "mix", | |
| "prefix": "dolma3-mix-19", | |
| "shards": 1, | |
| "tokens": 125000559, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v1", | |
| "prefix": "dolma3-smoke-v1", | |
| "shards": 1, | |
| "tokens": 49900787, | |
| "tokens_available": 49900787, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "finemath", | |
| "labels": [ | |
| "pretrain", | |
| "fresh5b", | |
| "finemath", | |
| "topup", | |
| "math", | |
| "phase6" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "5b", | |
| "prefix": "finemath5b-00", | |
| "shards": 7, | |
| "tokens": 853496772, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "finemath5b-01", | |
| "shards": 7, | |
| "tokens": 859288822, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "finemath5b-t0", | |
| "shards": 14, | |
| "tokens": 1707481779, | |
| "tokens_available": 1707481779, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "finemath5b-t1", | |
| "shards": 14, | |
| "tokens": 1710258536, | |
| "tokens_available": 1710258536, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v6", | |
| "prefix": "finemath6b", | |
| "shards": 6, | |
| "tokens": 683904776, | |
| "tokens_available": 683904776, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "fw2hq", | |
| "labels": [ | |
| "pretrain", | |
| "fresh5b", | |
| "FineWeb2-HQ", | |
| "topup" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "5b", | |
| "prefix": "fw2hq5b-00", | |
| "shards": 7, | |
| "tokens": 845050715, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "fw2hq5b-01", | |
| "shards": 4, | |
| "tokens": 427538262, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "fw2hq5b-02", | |
| "shards": 6, | |
| "tokens": 642160323, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "fw2hq5b-t0", | |
| "shards": 10, | |
| "tokens": 1136907489, | |
| "tokens_available": 1136907489, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "fw2hq5b-t1", | |
| "shards": 8, | |
| "tokens": 991812123, | |
| "tokens_available": 991812123, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "fw2hq5b-t2", | |
| "shards": 10, | |
| "tokens": 1132345967, | |
| "tokens_available": 1132345967, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "fwedu", | |
| "labels": [ | |
| "pretrain", | |
| "fresh5b", | |
| "fineweb-edu", | |
| "phase6" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "5b", | |
| "prefix": "fwedu5b-00", | |
| "shards": 14, | |
| "tokens": 1653806023, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "fwedu5b-01", | |
| "shards": 13, | |
| "tokens": 1588759282, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "fwedu5b-03", | |
| "shards": 14, | |
| "tokens": 1675779916, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "5b", | |
| "prefix": "fwedu5b-04", | |
| "shards": 7, | |
| "tokens": 825807526, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v6", | |
| "prefix": "fwedu6b", | |
| "shards": 21, | |
| "tokens": 2526559766, | |
| "tokens_available": 2526559766, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v6", | |
| "prefix": "fwedu6b-t1", | |
| "shards": 32, | |
| "tokens": 4000056044, | |
| "tokens_available": 4000056044, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "kaggle", | |
| "labels": [ | |
| "pretrain", | |
| "fresh5b", | |
| "issues-kaggle-notebooks" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "5b", | |
| "prefix": "kaggle5b-00", | |
| "shards": 40, | |
| "tokens": 5000030082, | |
| "tokens_available": 3625025011, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "megamath", | |
| "labels": [ | |
| "math", | |
| "web", | |
| "phase6" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "v6", | |
| "prefix": "megamath6b", | |
| "shards": 7, | |
| "tokens": 850278779, | |
| "tokens_available": 850278779, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v6", | |
| "prefix": "megamath6b-t1", | |
| "shards": 10, | |
| "tokens": 1225016936, | |
| "tokens_available": 1225016936, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "openmathinstruct2", | |
| "labels": [ | |
| "math", | |
| "solutions", | |
| "phase6" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "v6", | |
| "prefix": "omi2-6b-t0", | |
| "shards": 4, | |
| "tokens": 414455542, | |
| "tokens_available": 414455542, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v6", | |
| "prefix": "omi2-6b-t1", | |
| "shards": 4, | |
| "tokens": 424561740, | |
| "tokens_available": 424561740, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "python-edu", | |
| "labels": [ | |
| "code", | |
| "python", | |
| "edu", | |
| "phase6" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "v6", | |
| "prefix": "pyedu6b", | |
| "shards": 10, | |
| "tokens": 1167242245, | |
| "tokens_available": 1167242245, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "tokforge-test", | |
| "labels": [ | |
| "smoke", | |
| "pretrain", | |
| "e2e", | |
| "resume-test" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "v1", | |
| "prefix": "tokforge-test-v1", | |
| "shards": 3, | |
| "tokens": 600606, | |
| "tokens_available": 600606, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v12", | |
| "prefix": "tokforge-test-v12", | |
| "shards": 3, | |
| "tokens": 600606, | |
| "tokens_available": 600606, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v13", | |
| "prefix": "tokforge-test-v13", | |
| "shards": 3, | |
| "tokens": 600606, | |
| "tokens_available": 600606, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v14", | |
| "prefix": "tokforge-test-v14", | |
| "shards": 3, | |
| "tokens": 600606, | |
| "tokens_available": 600606, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v2", | |
| "prefix": "tokforge-test-v2", | |
| "shards": 3, | |
| "tokens": 600606, | |
| "tokens_available": 600606, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v3", | |
| "prefix": "tokforge-test-v3", | |
| "shards": 3, | |
| "tokens": 600606, | |
| "tokens_available": 200031, | |
| "tokens_consumed": 200502 | |
| }, | |
| { | |
| "version": "v4", | |
| "prefix": "tokforge-test-v4", | |
| "shards": 3, | |
| "tokens": 600606, | |
| "tokens_available": 600606, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v5", | |
| "prefix": "tokforge-test-v5", | |
| "shards": 5, | |
| "tokens": 1001649, | |
| "tokens_available": 1001649, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v6", | |
| "prefix": "tokforge-test-v6", | |
| "shards": 15, | |
| "tokens": 3003892, | |
| "tokens_available": 3003892, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v7", | |
| "prefix": "tokforge-test-v7", | |
| "shards": 3, | |
| "tokens": 600606, | |
| "tokens_available": 600606, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "webinstruct", | |
| "labels": [ | |
| "reasoning", | |
| "qa", | |
| "phase6" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "v6", | |
| "prefix": "webinstr6b", | |
| "shards": 5, | |
| "tokens": 510487802, | |
| "tokens_available": 510487802, | |
| "tokens_consumed": 0 | |
| }, | |
| { | |
| "version": "v6", | |
| "prefix": "webinstr6b-t1", | |
| "shards": 6, | |
| "tokens": 750002235, | |
| "tokens_available": 0, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| }, | |
| { | |
| "corpus": "wikipedia", | |
| "labels": [ | |
| "facts", | |
| "encyclopedia", | |
| "phase6" | |
| ], | |
| "versions": [ | |
| { | |
| "version": "v6", | |
| "prefix": "wiki6b", | |
| "shards": 8, | |
| "tokens": 1000008988, | |
| "tokens_available": 1000008988, | |
| "tokens_consumed": 0 | |
| } | |
| ] | |
| } | |
| ], | |
| "shards_total": 702, | |
| "tokens_total": 68028933571, | |
| "tokens_available": 32164429255, | |
| "tokens_consumed": 200502 | |
| } |
Xet Storage Details
- Size:
- 19 kB
- Xet hash:
- 742cb4dda7525678f768888b1ae89087de6698573053349346da75a0111d6966
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.