{
  "total_loaded": 1239240,
  "total_after_lang_filter": 755703,
  "total_after_dedup": 661014,
  "total_tokens": 24704887,
  "total_training_samples": 661008,
  "total_instruction": 95437,
  "total_chat": 0,
  "per_file_stats": [
    {
      "file": "huggingface_ChrisToukmaji_kinyarwanda_instruction_tuning.jsonl",
      "loaded": 4250,
      "lang_filtered": 2,
      "clean_filtered": 0,
      "kept": 4248,
      "tokens": 327390,
      "instruction_samples": 4248,
      "chat_samples": 0
    },
    {
      "file": "huggingface_CircuitNotion_kinyarwanda_corpus.jsonl",
      "loaded": 0,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 0,
      "tokens": 0,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_DigitalUmuganda_kinyarwanda-english-machine-translation-dataset.jsonl",
      "loaded": 0,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 0,
      "tokens": 0,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_DigitalUmuganda_NMT_Health_parallel_data_en_kin.jsonl",
      "loaded": 30000,
      "lang_filtered": 1,
      "clean_filtered": 4274,
      "kept": 25725,
      "tokens": 524882,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_DigitalUmuganda_NMT_Rwandan-Gazette_parallel_data_en_kin.jsonl",
      "loaded": 80000,
      "lang_filtered": 0,
      "clean_filtered": 9768,
      "kept": 70232,
      "tokens": 1727805,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_jojo-ai-mst_Roleplay-Kinyarwanda.jsonl",
      "loaded": 1923,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 1923,
      "tokens": 50283,
      "instruction_samples": 1923,
      "chat_samples": 0
    },
    {
      "file": "huggingface_mbazaNLP_Kinyarwanda_English_parallel_dataset.jsonl",
      "loaded": 55667,
      "lang_filtered": 0,
      "clean_filtered": 7384,
      "kept": 48283,
      "tokens": 779663,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_michsethowusu_akan-kinyarwanda_sentence-pairs.jsonl",
      "loaded": 67145,
      "lang_filtered": 7800,
      "clean_filtered": 39704,
      "kept": 19641,
      "tokens": 294505,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_michsethowusu_bambara-kinyarwanda_sentence-pairs.jsonl",
      "loaded": 56301,
      "lang_filtered": 3366,
      "clean_filtered": 25937,
      "kept": 26998,
      "tokens": 522282,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_michsethowusu_bemba-kinyarwanda_sentence-pairs.jsonl",
      "loaded": 197886,
      "lang_filtered": 2,
      "clean_filtered": 101878,
      "kept": 96006,
      "tokens": 1379707,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_michsethowusu_dinka-kinyarwanda_sentence-pairs.jsonl",
      "loaded": 39196,
      "lang_filtered": 6663,
      "clean_filtered": 18172,
      "kept": 14361,
      "tokens": 268754,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_michsethowusu_dyula-kinyarwanda_sentence-pairs.jsonl",
      "loaded": 102925,
      "lang_filtered": 2446,
      "clean_filtered": 57811,
      "kept": 42668,
      "tokens": 857116,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_michsethowusu_kinyarwanda-rundi_sentence-pairs.jsonl",
      "loaded": 410000,
      "lang_filtered": 11,
      "clean_filtered": 163087,
      "kept": 246902,
      "tokens": 3516163,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_Mikecyane_Kinyarwanda_chat.jsonl",
      "loaded": 0,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 0,
      "tokens": 0,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_RogerB_kin_en_DigitalUmuganda.jsonl",
      "loaded": 47824,
      "lang_filtered": 0,
      "clean_filtered": 30920,
      "kept": 16904,
      "tokens": 159881,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_saillab_alpaca-kinyarwanda-cleaned.jsonl",
      "loaded": 41601,
      "lang_filtered": 2,
      "clean_filtered": 1900,
      "kept": 39699,
      "tokens": 1440835,
      "instruction_samples": 41599,
      "chat_samples": 0
    },
    {
      "file": "huggingface_saillab_alpaca_kinyarwanda_taco.jsonl",
      "loaded": 49601,
      "lang_filtered": 11,
      "clean_filtered": 0,
      "kept": 49590,
      "tokens": 12116090,
      "instruction_samples": 49590,
      "chat_samples": 0
    },
    {
      "file": "huggingface_souvorinkg_english_kinyarwanda.jsonl",
      "loaded": 54589,
      "lang_filtered": 0,
      "clean_filtered": 2387,
      "kept": 52202,
      "tokens": 662665,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "igihe_igihe_articles.jsonl",
      "loaded": 0,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 0,
      "tokens": 0,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "kigalitoday_kigalitoday_articles.jsonl",
      "loaded": 0,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 0,
      "tokens": 0,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "newtimes_rss_newtimes_rss_articles.jsonl",
      "loaded": 0,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 0,
      "tokens": 0,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "rbc_rbc_articles.jsonl",
      "loaded": 0,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 0,
      "tokens": 0,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "wikipedia_wikipedia_articles.jsonl",
      "loaded": 274,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 274,
      "tokens": 75991,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "wikisource_wikisource_articles.jsonl",
      "loaded": 0,
      "lang_filtered": 0,
      "clean_filtered": 0,
      "kept": 0,
      "tokens": 0,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "wiktionary_wiktionary_entries.jsonl",
      "loaded": 58,
      "lang_filtered": 11,
      "clean_filtered": 0,
      "kept": 47,
      "tokens": 875,
      "instruction_samples": 0,
      "chat_samples": 0
    }
  ],
  "output_files": {
    "training": "data\\processed\\training_data.jsonl",
    "instruction": "data\\processed\\instruction_data.jsonl",
    "chat": "data\\processed\\chat_data.jsonl"
  }
}