tokenizer_test4 / evaluation /evaluation_summary.json
ljcamargo's picture
Upload evaluation/evaluation_summary.json with huggingface_hub
d1417c8 verified
Raw
History Blame Contribute Delete
453 Bytes
{
"catalogue_url": "https://huggingface.co/datasets/tachiwin/language_utils/resolve/main/catalogue.json",
"languages_in_catalogue": 363,
"languages_with_samples": 106,
"samples_evaluated": 327,
"failed_sample_round_trips": 0,
"failed_language_round_trips": 0,
"mean_chars_per_token": 3.414915883033258,
"median_chars_per_token": 3.3048122107830826,
"min_chars_per_token": 2.289772727272727,
"max_chars_per_token": 5.385026737967914
}