File size: 889 Bytes
8662ab2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | {
"fixtures_passed": 9,
"sources": {
"cosmopedia": {
"bytes": 402259,
"bytes_per_token": 5.061007523716062,
"documents": 103,
"roundtrip_failures": 0,
"seconds": 0.09366637503262609,
"tokens": 79482
},
"dclm": {
"bytes": 1001509,
"bytes_per_token": 4.453209483494593,
"documents": 165,
"roundtrip_failures": 0,
"seconds": 0.26016941701527685,
"tokens": 224896
},
"web": {
"bytes": 2205062,
"bytes_per_token": 4.625467253451697,
"documents": 559,
"roundtrip_failures": 0,
"seconds": 0.5487007499905303,
"tokens": 476722
},
"wiki": {
"bytes": 401775,
"bytes_per_token": 4.059276397546905,
"documents": 143,
"roundtrip_failures": 0,
"seconds": 0.10723229206632823,
"tokens": 98977
}
},
"vocab_size": 32000
}
|