Upload .\scripts\collect_all.py with huggingface_hub

.//scripts//collect_all.py

@@ -0,0 +1,293 @@
+"""Main data collection script for BwengeAi."""
+
+import json
+import logging
+import sys
+from pathlib import Path
+
+import yaml
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+
+sys.path.insert(0, str(PROJECT_ROOT / "src"))
+
+from data_collection.huggingface_collector import HuggingfaceCollector
+from data_collection.parallel_corpus_collector import ParallelCorpusCollector
+from data_collection.cc100_oscar_collector import CC100OSCARCollector
+from data_collection.masakhane_collector import MasakhaneCollector
+from data_collection.wikipedia_collector import WikipediaCollector
+from data_collection.igihe_scraper import IgiheScraper
+from data_collection.kigalitoday_scraper import KigaliTodayScraper
+from data_collection.rss_collector import RSSCollector
+from data_collection.wikimedia_collector import WikisourceCollector, WiktionaryCollector
+from data_collection.rbc_collector import RBCCollector
+from data_collection.data_processor import DataProcessor
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+def load_config(config_path: str = None) -> dict:
+    """Load configuration."""
+    if config_path is None:
+        config_path = str(PROJECT_ROOT / "configs/default.yaml")
+    with open(config_path, "r", encoding="utf-8") as f:
+        return yaml.safe_load(f)
+
+
+def main():
+    """Run data collection pipeline."""
+    logger.info("=" * 60)
+    logger.info("BwengeAi Data Collection Pipeline")
+    logger.info("=" * 60)
+
+    config = load_config()
+    data_config = config.get("data", {})
+
+    raw_dir = str(PROJECT_ROOT / data_config.get("raw_dir", "data/raw"))
+    processed_dir = str(PROJECT_ROOT / data_config.get("processed_dir", "data/processed"))
+
+    hf_results = []
+    parallel_results = []
+    cc100_oscar_results = []
+    masakhane_results = []
+    wiki_results = []
+    igihe_results = []
+    kt_results = []
+    rss_results = []
+    wikisource_results = []
+    wiktionary_results = []
+    rbc_results = []
+
+    # Step 1: HuggingFace (main datasets)
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 1: Collecting from Huggingface")
+    logger.info("=" * 40)
+
+    try:
+        hf_collector = HuggingfaceCollector(
+            output_dir=f"{raw_dir}/huggingface",
+            config=data_config,
+        )
+        hf_results = hf_collector.collect_all()
+    except Exception as e:
+        logger.error(f"Huggingface collection failed: {e}")
+
+    # Step 2: Massive parallel corpora (michsethowusu)
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 2: Collecting massive parallel corpora")
+    logger.info("=" * 40)
+
+    try:
+        parallel_collector = ParallelCorpusCollector(
+            output_dir=f"{raw_dir}/parallel",
+            config=data_config,
+        )
+        parallel_results = parallel_collector.collect_all()
+    except Exception as e:
+        logger.error(f"Parallel corpus collection failed: {e}")
+
+    # Step 3: CC-100 and OSCAR monolingual
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 3: Collecting CC-100 and OSCAR monolingual corpora")
+    logger.info("=" * 40)
+
+    try:
+        cc100_collector = CC100OSCARCollector(
+            output_dir=f"{raw_dir}/cc100_oscar",
+            config=data_config,
+        )
+        cc100_oscar_results = cc100_collector.collect_all(languages=["rw"])
+    except Exception as e:
+        logger.error(f"CC-100/OSCAR collection failed: {e}")
+
+    # Step 4: Masakhane benchmarks
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 4: Collecting Masakhane benchmark datasets")
+    logger.info("=" * 40)
+
+    try:
+        masakhane_collector = MasakhaneCollector(
+            output_dir=f"{raw_dir}/masakhane",
+            config=data_config,
+        )
+        masakhane_results = masakhane_collector.collect_all()
+    except Exception as e:
+        logger.error(f"Masakhane collection failed: {e}")
+
+    # Step 5: Wikipedia (API + Dump)
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 5: Collecting from Wikipedia")
+    logger.info("=" * 40)
+
+    try:
+        wiki_collector = WikipediaCollector(
+            output_dir=f"{raw_dir}/wikipedia",
+            config=data_config,
+        )
+
+        stats = wiki_collector.get_site_stats()
+        logger.info(f"Wikipedia stats: {stats}")
+
+        wiki_results = wiki_collector.collect_via_api()
+
+        dump_path = wiki_collector.download_dump()
+        if dump_path:
+            dump_articles = wiki_collector.parse_dump(dump_path)
+            logger.info(f"Parsed {len(dump_articles)} articles from dump")
+    except Exception as e:
+        logger.error(f"Wikipedia collection failed: {e}")
+
+    # Step 6: Igihe
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 6: Collecting from Igihe (if permitted)")
+    logger.info("=" * 40)
+
+    try:
+        igihe_config = data_config.get("igihe", {})
+        contact_email = igihe_config.get("contact_email", "info@igihe.com")
+        logger.info(f"NOTE: Igihe content is copyrighted.")
+        logger.info(f"Contact {contact_email} for data licensing.")
+
+        igihe_scraper = IgiheScraper(
+            output_dir=f"{raw_dir}/igihe",
+            config=data_config,
+        )
+        igihe_results = igihe_scraper.collect_all()
+    except Exception as e:
+        logger.warning(f"Igihe collection failed: {e}")
+
+    # Step 7: Kigali Today
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 7: Collecting from Kigali Today")
+    logger.info("=" * 40)
+
+    try:
+        kt_scraper = KigaliTodayScraper(
+            output_dir=f"{raw_dir}/kigalitoday",
+            config=data_config,
+        )
+        kt_results = kt_scraper.collect_all()
+    except Exception as e:
+        logger.warning(f"Kigali Today collection failed: {e}")
+
+    # Step 8: RSS Feeds (New Times)
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 8: Collecting from RSS feeds (New Times)")
+    logger.info("=" * 40)
+
+    try:
+        rss_collector = RSSCollector(
+            output_dir=f"{raw_dir}/newtimes_rss",
+            config=data_config,
+        )
+        rss_results = rss_collector.collect_all()
+    except Exception as e:
+        logger.warning(f"RSS collection failed: {e}")
+
+    # Step 9: Wikisource
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 9: Collecting from Wikisource")
+    logger.info("=" * 40)
+
+    try:
+        ws_collector = WikisourceCollector(
+            output_dir=f"{raw_dir}/wikisource",
+            config=data_config,
+        )
+        wikisource_results = ws_collector.collect_all()
+    except Exception as e:
+        logger.warning(f"Wikisource collection failed: {e}")
+
+    # Step 10: Wiktionary
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 10: Collecting from Wiktionary")
+    logger.info("=" * 40)
+
+    try:
+        wt_collector = WiktionaryCollector(
+            output_dir=f"{raw_dir}/wiktionary",
+            config=data_config,
+        )
+        wiktionary_results = wt_collector.collect_all()
+    except Exception as e:
+        logger.warning(f"Wiktionary collection failed: {e}")
+
+    # Step 11: RBC
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 11: Collecting from RBC")
+    logger.info("=" * 40)
+
+    try:
+        rbc_collector = RBCCollector(
+            output_dir=f"{raw_dir}/rbc",
+            config=data_config,
+        )
+        rbc_results = rbc_collector.collect_all()
+    except Exception as e:
+        logger.warning(f"RBC collection failed: {e}")
+
+    # Step 12: Process and combine
+    logger.info("\n" + "=" * 40)
+    logger.info("Step 12: Processing and combining data")
+    logger.info("=" * 40)
+
+    processor = DataProcessor(
+        raw_dir=raw_dir,
+        processed_dir=processed_dir,
+    )
+
+    import shutil
+    seen_names: set[str] = set()
+    for jsonl_file in Path(raw_dir).rglob("*.jsonl"):
+        if jsonl_file.parent == Path(raw_dir):
+            continue
+        parent_dir = jsonl_file.parent.name
+        prefixed_name = f"{parent_dir}_{jsonl_file.name}"
+        dest = Path(raw_dir) / prefixed_name
+        if prefixed_name not in seen_names:
+            shutil.copy2(jsonl_file, dest)
+            seen_names.add(prefixed_name)
+
+    processing_summary = processor.process_all()
+
+    logger.info("\n" + "=" * 60)
+    logger.info("Data Collection Complete!")
+    logger.info("=" * 60)
+
+    summary = {
+        "huggingface_datasets": len(hf_results),
+        "huggingface_successful": sum(1 for r in hf_results if r.get("status") == "success"),
+        "parallel_datasets": len(parallel_results),
+        "parallel_successful": sum(1 for r in parallel_results if r.get("status") == "success"),
+        "parallel_total_rows": sum(r.get("rows", 0) for r in parallel_results if r.get("status") == "success"),
+        "cc100_oscar_datasets": len(cc100_oscar_results),
+        "cc100_oscar_successful": sum(1 for r in cc100_oscar_results if r.get("status") == "success"),
+        "masakhane_datasets": len(masakhane_results),
+        "masakhane_successful": sum(1 for r in masakhane_results if r.get("status") == "success"),
+        "wikipedia_articles": len(wiki_results),
+        "igihe_articles": len(igihe_results),
+        "kigalitoday_articles": len(kt_results),
+        "newtimes_rss_articles": len(rss_results),
+        "wikisource_texts": len(wikisource_results),
+        "wiktionary_entries": len(wiktionary_results),
+        "rbc_articles": len(rbc_results),
+        "processing": processing_summary,
+    }
+
+    summary_path = Path(processed_dir) / "collection_summary.json"
+    with open(summary_path, "w", encoding="utf-8") as f:
+        json.dump(summary, f, indent=2, ensure_ascii=False)
+
+    logger.info(f"Summary saved to {summary_path}")
+    logger.info(f"Total tokens: {processing_summary.get('total_tokens', 0):,}")
+    logger.info(f"Total training samples: {processing_summary.get('total_training_samples', 0):,}")
+    logger.info(f"Total instruction samples: {processing_summary.get('total_instruction', 0):,}")
+    logger.info(f"Total chat samples: {processing_summary.get('total_chat', 0):,}")
+
+
+if __name__ == "__main__":
+    main()