| { | |
| "total_loaded": 1239240, | |
| "total_after_lang_filter": 755703, | |
| "total_after_dedup": 661014, | |
| "total_tokens": 24704887, | |
| "total_training_samples": 661008, | |
| "total_instruction": 95437, | |
| "total_chat": 0, | |
| "per_file_stats": [ | |
| { | |
| "file": "huggingface_ChrisToukmaji_kinyarwanda_instruction_tuning.jsonl", | |
| "loaded": 4250, | |
| "lang_filtered": 2, | |
| "clean_filtered": 0, | |
| "kept": 4248, | |
| "tokens": 327390, | |
| "instruction_samples": 4248, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_CircuitNotion_kinyarwanda_corpus.jsonl", | |
| "loaded": 0, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 0, | |
| "tokens": 0, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_DigitalUmuganda_kinyarwanda-english-machine-translation-dataset.jsonl", | |
| "loaded": 0, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 0, | |
| "tokens": 0, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_DigitalUmuganda_NMT_Health_parallel_data_en_kin.jsonl", | |
| "loaded": 30000, | |
| "lang_filtered": 1, | |
| "clean_filtered": 4274, | |
| "kept": 25725, | |
| "tokens": 524882, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_DigitalUmuganda_NMT_Rwandan-Gazette_parallel_data_en_kin.jsonl", | |
| "loaded": 80000, | |
| "lang_filtered": 0, | |
| "clean_filtered": 9768, | |
| "kept": 70232, | |
| "tokens": 1727805, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_jojo-ai-mst_Roleplay-Kinyarwanda.jsonl", | |
| "loaded": 1923, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 1923, | |
| "tokens": 50283, | |
| "instruction_samples": 1923, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_mbazaNLP_Kinyarwanda_English_parallel_dataset.jsonl", | |
| "loaded": 55667, | |
| "lang_filtered": 0, | |
| "clean_filtered": 7384, | |
| "kept": 48283, | |
| "tokens": 779663, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_michsethowusu_akan-kinyarwanda_sentence-pairs.jsonl", | |
| "loaded": 67145, | |
| "lang_filtered": 7800, | |
| "clean_filtered": 39704, | |
| "kept": 19641, | |
| "tokens": 294505, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_michsethowusu_bambara-kinyarwanda_sentence-pairs.jsonl", | |
| "loaded": 56301, | |
| "lang_filtered": 3366, | |
| "clean_filtered": 25937, | |
| "kept": 26998, | |
| "tokens": 522282, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_michsethowusu_bemba-kinyarwanda_sentence-pairs.jsonl", | |
| "loaded": 197886, | |
| "lang_filtered": 2, | |
| "clean_filtered": 101878, | |
| "kept": 96006, | |
| "tokens": 1379707, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_michsethowusu_dinka-kinyarwanda_sentence-pairs.jsonl", | |
| "loaded": 39196, | |
| "lang_filtered": 6663, | |
| "clean_filtered": 18172, | |
| "kept": 14361, | |
| "tokens": 268754, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_michsethowusu_dyula-kinyarwanda_sentence-pairs.jsonl", | |
| "loaded": 102925, | |
| "lang_filtered": 2446, | |
| "clean_filtered": 57811, | |
| "kept": 42668, | |
| "tokens": 857116, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_michsethowusu_kinyarwanda-rundi_sentence-pairs.jsonl", | |
| "loaded": 410000, | |
| "lang_filtered": 11, | |
| "clean_filtered": 163087, | |
| "kept": 246902, | |
| "tokens": 3516163, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_Mikecyane_Kinyarwanda_chat.jsonl", | |
| "loaded": 0, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 0, | |
| "tokens": 0, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_RogerB_kin_en_DigitalUmuganda.jsonl", | |
| "loaded": 47824, | |
| "lang_filtered": 0, | |
| "clean_filtered": 30920, | |
| "kept": 16904, | |
| "tokens": 159881, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_saillab_alpaca-kinyarwanda-cleaned.jsonl", | |
| "loaded": 41601, | |
| "lang_filtered": 2, | |
| "clean_filtered": 1900, | |
| "kept": 39699, | |
| "tokens": 1440835, | |
| "instruction_samples": 41599, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_saillab_alpaca_kinyarwanda_taco.jsonl", | |
| "loaded": 49601, | |
| "lang_filtered": 11, | |
| "clean_filtered": 0, | |
| "kept": 49590, | |
| "tokens": 12116090, | |
| "instruction_samples": 49590, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_souvorinkg_english_kinyarwanda.jsonl", | |
| "loaded": 54589, | |
| "lang_filtered": 0, | |
| "clean_filtered": 2387, | |
| "kept": 52202, | |
| "tokens": 662665, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "igihe_igihe_articles.jsonl", | |
| "loaded": 0, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 0, | |
| "tokens": 0, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "kigalitoday_kigalitoday_articles.jsonl", | |
| "loaded": 0, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 0, | |
| "tokens": 0, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "newtimes_rss_newtimes_rss_articles.jsonl", | |
| "loaded": 0, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 0, | |
| "tokens": 0, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "rbc_rbc_articles.jsonl", | |
| "loaded": 0, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 0, | |
| "tokens": 0, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "wikipedia_wikipedia_articles.jsonl", | |
| "loaded": 274, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 274, | |
| "tokens": 75991, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "wikisource_wikisource_articles.jsonl", | |
| "loaded": 0, | |
| "lang_filtered": 0, | |
| "clean_filtered": 0, | |
| "kept": 0, | |
| "tokens": 0, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "wiktionary_wiktionary_entries.jsonl", | |
| "loaded": 58, | |
| "lang_filtered": 11, | |
| "clean_filtered": 0, | |
| "kept": 47, | |
| "tokens": 875, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| } | |
| ], | |
| "output_files": { | |
| "training": "data\\processed\\training_data.jsonl", | |
| "instruction": "data\\processed\\instruction_data.jsonl", | |
| "chat": "data\\processed\\chat_data.jsonl" | |
| } | |
| } |