Upload .\scripts\collect_huggingface.py with huggingface_hub

.//scripts//collect_huggingface.py

@@ -0,0 +1,79 @@
+"""Standalone Huggingface collector - downloads datasets as JSONL without torch dependency."""
+
+import json
+import logging
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+OUTPUT_DIR = Path("C:/Users/admin/BwengeAi/data/raw/huggingface")
+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+# Key datasets to collect (streaming mode to avoid loading all in memory)
+DATASETS = [
+    {"name": "mbazaNLP/kinyarwanda_monolingual_v01.1", "desc": "Monolingual corpus (1.07M rows)"},
+    {"name": "CircuitNotion/kinyarwanda_corpus", "desc": "Large corpus (3.52M rows)"},
+    {"name": "saillab/alpaca_kinyarwanda_taco", "desc": "Instruction tuning (62k rows)"},
+    {"name": "saillab/alpaca-kinyarwanda-cleaned", "desc": "Cleaned instruction (52k rows)"},
+    {"name": "mbazaNLP/Kinyarwanda_English_parallel_dataset", "desc": "Parallel corpus (55.7k rows)"},
+    {"name": "ChrisToukmaji/kinyarwanda_instruction_tuning", "desc": "Instruction tuning (5k rows)"},
+    {"name": "Mikecyane/Kinyarwanda_chat", "desc": "Chat dataset (4.16k rows)"},
+    {"name": "RogerB/Kinyarwanda_wikipedia20230920", "desc": "Wikipedia dump (8.05k rows)"},
+    {"name": "tianyiwordly/kinyarwanda_denoised", "desc": "Denoised text (783k rows)"},
+    {"name": "michsethowusu/Code-170k-kinyarwanda", "desc": "Code in Kinyarwanda (177k rows)"},
+    {"name": "FarmerlineML/kinyarwanda_dataset", "desc": "Agriculture Kinyarwanda"},
+    {"name": "DigitalUmuganda/kinyarwanda-english-machine-translation-dataset", "desc": "MT dataset (3.38k rows)"},
+]
+
+
+def collect_dataset(dataset_config: dict) -> dict:
+    from datasets import load_dataset
+
+    name = dataset_config["name"]
+    safe_name = name.replace("/", "_")
+    output_path = OUTPUT_DIR / f"{safe_name}.jsonl"
+
+    logger.info(f"Collecting: {name} ({dataset_config['desc']})")
+
+    try:
+        dataset = load_dataset(name, split="train", streaming=True, trust_remote_code=True)
+
+        count = 0
+        with open(output_path, "w", encoding="utf-8") as f:
+            for item in dataset:
+                f.write(json.dumps(item, ensure_ascii=False) + "\n")
+                count += 1
+                if count % 10000 == 0:
+                    logger.info(f"  {count} rows collected...")
+
+        logger.info(f"  Saved {count} rows to {output_path}")
+        return {"name": name, "rows": count, "path": str(output_path), "status": "success"}
+
+    except Exception as e:
+        logger.error(f"  Failed: {e}")
+        return {"name": name, "status": "failed", "error": str(e)}
+
+
+def main():
+    logger.info("=" * 60)
+    logger.info("BwengeAi - Huggingface Dataset Collection")
+    logger.info("=" * 60)
+
+    results = []
+    for ds_config in DATASETS:
+        result = collect_dataset(ds_config)
+        results.append(result)
+
+    summary_path = OUTPUT_DIR / "huggingface_summary.json"
+    with open(summary_path, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+
+    successful = sum(1 for r in results if r.get("status") == "success")
+    total_rows = sum(r.get("rows", 0) for r in results if r.get("status") == "success")
+    logger.info(f"\nDone! {successful}/{len(results)} datasets collected, {total_rows} total rows")
+    logger.info(f"Summary: {summary_path}")
+
+
+if __name__ == "__main__":
+    main()