{
  "total_loaded": 176001,
  "total_after_lang_filter": 166586,
  "total_after_dedup": 154569,
  "total_tokens": 29529921,
  "total_training_samples": 154565,
  "total_instruction": 41599,
  "total_chat": 0,
  "per_file_stats": [
    {
      "file": "huggingface_mbazaNLP_Kinyarwanda_English_parallel_dataset.jsonl",
      "loaded": 55667,
      "lang_filtered": 0,
      "clean_filtered": 7384,
      "kept": 48283,
      "tokens": 779663,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_mbazaNLP_kinyarwanda_monolingual_v01.0.jsonl",
      "loaded": 78733,
      "lang_filtered": 129,
      "clean_filtered": 0,
      "kept": 78604,
      "tokens": 27309423,
      "instruction_samples": 0,
      "chat_samples": 0
    },
    {
      "file": "huggingface_saillab_alpaca-kinyarwanda-cleaned.jsonl",
      "loaded": 41601,
      "lang_filtered": 2,
      "clean_filtered": 1900,
      "kept": 39699,
      "tokens": 1440835,
      "instruction_samples": 41599,
      "chat_samples": 0
    }
  ],
  "output_files": {
    "training": "C:\\Users\\admin\\BwengeAi\\data\\processed_test\\training_data.jsonl",
    "instruction": "C:\\Users\\admin\\BwengeAi\\data\\processed_test\\instruction_data.jsonl",
    "chat": "C:\\Users\\admin\\BwengeAi\\data\\processed_test\\chat_data.jsonl"
  }
}