| { | |
| "vocab_size": 49152, | |
| "num_special_tokens": 33, | |
| "ratios": { | |
| "ita": 0.4, | |
| "en": 0.4, | |
| "code": 0.2 | |
| }, | |
| "char_budget": 4000000000, | |
| "special_token_ids": { | |
| "bos": 49119, | |
| "eos": 49120, | |
| "pad": 49121 | |
| }, | |
| "pad_distinct_from_eos": true, | |
| "train_time_sec": 126.83898162841797, | |
| "sanity": [ | |
| { | |
| "text": "Ciao mondo, l'Italia \u00e8 bellissima.", | |
| "num_tokens": 8, | |
| "roundtrip_ok": true | |
| }, | |
| { | |
| "text": "The quick brown fox jumps over the lazy dog.", | |
| "num_tokens": 11, | |
| "roundtrip_ok": true | |
| }, | |
| { | |
| "text": "def add(a, b):\n return a + b # sum", | |
| "num_tokens": 14, | |
| "roundtrip_ok": true | |
| } | |
| ] | |
| } |