| { | |
| "total_loaded": 176001, | |
| "total_after_lang_filter": 166586, | |
| "total_after_dedup": 154569, | |
| "total_tokens": 29529921, | |
| "total_training_samples": 154565, | |
| "total_instruction": 41599, | |
| "total_chat": 0, | |
| "per_file_stats": [ | |
| { | |
| "file": "huggingface_mbazaNLP_Kinyarwanda_English_parallel_dataset.jsonl", | |
| "loaded": 55667, | |
| "lang_filtered": 0, | |
| "clean_filtered": 7384, | |
| "kept": 48283, | |
| "tokens": 779663, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_mbazaNLP_kinyarwanda_monolingual_v01.0.jsonl", | |
| "loaded": 78733, | |
| "lang_filtered": 129, | |
| "clean_filtered": 0, | |
| "kept": 78604, | |
| "tokens": 27309423, | |
| "instruction_samples": 0, | |
| "chat_samples": 0 | |
| }, | |
| { | |
| "file": "huggingface_saillab_alpaca-kinyarwanda-cleaned.jsonl", | |
| "loaded": 41601, | |
| "lang_filtered": 2, | |
| "clean_filtered": 1900, | |
| "kept": 39699, | |
| "tokens": 1440835, | |
| "instruction_samples": 41599, | |
| "chat_samples": 0 | |
| } | |
| ], | |
| "output_files": { | |
| "training": "C:\\Users\\admin\\BwengeAi\\data\\processed_test\\training_data.jsonl", | |
| "instruction": "C:\\Users\\admin\\BwengeAi\\data\\processed_test\\instruction_data.jsonl", | |
| "chat": "C:\\Users\\admin\\BwengeAi\\data\\processed_test\\chat_data.jsonl" | |
| } | |
| } |