Upload .\scripts\count_stats.py with huggingface_hub

.//scripts//count_stats.py

@@ -0,0 +1,379 @@
+from pathlib import Path
+import yaml
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+d = yaml.safe_load(open(PROJECT_ROOT / "configs/default.yaml"))
+datasets = d["data"]["huggingface"]["datasets"]
+
+# Estimated rows and tokens per dataset (based on research findings)
+# Format: {name: (estimated_rows, estimated_tokens)}
+# Tokens estimated as ~4 chars per token for multilingual text
+
+dataset_stats = {
+    # === MONOLINGUAL KINYARWANDA ===
+    "CircuitNotion/kinyarwanda_corpus": (3_520_000, 1_500_000_000),
+    "mbazaNLP/kinyarwanda_monolingual_v01.1": (1_070_000, 520_000_000),
+    "mbazaNLP/kinyarwanda_monolingual_v01.0": (78_700, 50_000_000),
+
+    # === MASSIVE WEB CRAWL ===
+    "HuggingFaceFW/fineweb-2": (2_000_000_000, 20_000_000_000),  # kin subset estimated
+    "HPLT/HPLT2.0_cleaned": (50_000_000, 5_000_000_000),
+
+    # === OPUS PARALLEL ===
+    "Helsinki-NLP/opus-100": (178_000, 10_000_000),
+    "allenai/nllb": (500_000, 30_000_000),
+    "ahelk/ccaligned_multilingual": (100_000, 6_000_000),
+
+    # === INSTRUCTION TUNING ===
+    "saillab/alpaca_kinyarwanda_taco": (62_000, 15_000_000),
+    "saillab/alpaca-kinyarwanda-cleaned": (52_000, 8_000_000),
+    "ChrisToukmaji/kinyarwanda_instruction_tuning": (5_000, 1_500_000),
+
+    # === CHAT ===
+    "Mikecyane/Kinyarwanda_chat": (4_160, 1_000_000),
+    "jojo-ai-mst/Roleplay-Kinyarwanda": (1_920, 500_000),
+
+    # === KIN-EN PARALLEL ===
+    "mbazaNLP/Kinyarwanda_English_parallel_dataset": (55_700, 8_000_000),
+    "DigitalUmuganda/kinyarwanda-english-machine-translation-dataset": (48_000, 6_000_000),
+    "souvorinkg/english_kinyarwanda": (54_600, 7_000_000),
+    "RogerB/kin_en_DigitalUmuganda": (47_800, 5_500_000),
+    "DigitalUmuganda/NMT_Rwandan-Gazette_parallel_data_en_kin": (170_000, 25_000_000),
+    "DigitalUmuganda/NMT_Health_parallel_data_en_kin": (69_900, 12_000_000),
+
+    # === MASSIVE MULTILINGUAL PARALLEL (michsethowusu) ===
+    "michsethowusu/french-kinyarwanda_sentence-pairs": (2_660_000, 380_000_000),
+    "michsethowusu/afrikaans-kinyarwanda_sentence-pairs": (1_040_000, 150_000_000),
+    "michsethowusu/kinyarwanda-zulu_sentence-pairs": (546_000, 80_000_000),
+    "michsethowusu/amharic-kinyarwanda_sentence-pairs": (304_000, 50_000_000),
+    "michsethowusu/kinyarwanda-tswana_sentence-pairs": (282_000, 40_000_000),
+    "michsethowusu/ewe-kinyarwanda_sentence-pairs": (210_000, 30_000_000),
+    "michsethowusu/bemba-kinyarwanda_sentence-pairs": (198_000, 28_000_000),
+    "michsethowusu/dyula-kinyarwanda_sentence-pairs": (103_000, 15_000_000),
+    "michsethowusu/akan-kinyarwanda_sentence-pairs": (67_000, 9_000_000),
+    "michsethowusu/bambara-kinyarwanda_sentence-pairs": (56_000, 8_000_000),
+    "michsethowusu/dinka-kinyarwanda_sentence-pairs": (39_000, 5_500_000),
+    "michsethowusu/kinyarwanda-rundi_sentence-pairs": (80_000, 12_000_000),
+    "michsethowusu/kinyarwanda-shona_sentence-pairs": (75_000, 11_000_000),
+    "michsethowusu/kinyarwanda-xhosa_sentence-pairs": (60_000, 9_000_000),
+    "michsethowusu/kinyarwanda-tsonga_sentence-pairs": (50_000, 7_500_000),
+    "michsethowusu/kinyarwanda-yoruba_sentence-pairs": (85_000, 12_500_000),
+    "michsethowusu/kinyarwanda-somali_sentence-pairs": (45_000, 6_500_000),
+    "michsethowusu/kinyarwanda-pedi_sentence-pairs": (40_000, 6_000_000),
+    "michsethowusu/kinyarwanda-oromo_sentence-pairs": (55_000, 8_000_000),
+
+    # === WIKIPEDIA ===
+    "RogerB/Kinyarwanda_wikipedia20230920": (8_050, 3_000_000),
+
+    # === CODE ===
+    "michsethowusu/Code-170k-kinyarwanda": (177_000, 80_000_000),
+
+    # === HEALTH ===
+    "DigitalUmuganda/Monolingual_health_dataset": (71_200, 12_000_000),
+
+    # === NER ===
+    "masakhane/masakhaner2": (153_000, 5_000_000),
+    "masakhane/masakhaner": (3_000, 100_000),
+
+    # === QA ===
+    "masakhane/afriqa": (1_163, 200_000),
+    "mbazaNLP/PIQA-kin": (100, 15_000),
+
+    # === SENTIMENT ===
+    "masakhane/afrisenti": (5_155, 800_000),
+
+    # === NLI ===
+    "masakhane/afrixnli": (1_050, 200_000),
+
+    # === MMLU ===
+    "masakhane/afrimmlu": (608, 100_000),
+
+    # === INTENT ===
+    "masakhane/InjongoIntent": (3_200, 500_000),
+
+    # === MATH ===
+    "masakhane/afrimgsm": (2_000, 400_000),
+
+    # === DIALOGUE ===
+    "masakhane/AfriADR": (36_300, 5_000_000),
+
+    # === QA BENCHMARKS ===
+    "masakhane/uhura-arc-easy": (8_660, 1_500_000),
+    "masakhane/uhura-truthfulqa": (11_300, 2_000_000),
+
+    # === TRANSLATION BENCHMARKS ===
+    "facebook/flores": (2_000, 500_000),
+    "openlanguagedata/flores_plus": (883_000, 150_000_000),
+    "masakhane/mafand": (1_471, 300_000),
+
+    # === MULTILINGUAL AFRICAN ===
+    "LughaGen-Organization/LughaGen-Multilingual-African-Corpus": (732_000, 120_000_000),
+
+    # === NEWS ===
+    "Andrews2017/KINNEWS-and-KIRNEWS-Corpus": (21_000, 4_000_000),
+
+    # === COMMON VOICE ===
+    "mbazaNLP/common-voice-kinyarwanda-english-dataset": (721_000, 100_000_000),
+    "Kira-Floris/Afrivoice-Kinyarwanda-ASR": (566_000, 80_000_000),
+
+    # === MULTILINGUAL ALPACA ===
+    "saillab/alpaca_swahili_taco": (62_000, 10_000_000),
+    "saillab/alpaca_zulu_taco": (62_000, 10_000_000),
+    "saillab/alpaca_yoruba_taco": (62_000, 10_000_000),
+    "saillab/alpaca_igbo_taco": (62_000, 10_000_000),
+    "saillab/alpaca_hausa_taco": (62_000, 10_000_000),
+    "saillab/alpaca_shona_taco": (62_000, 10_000_000),
+    "saillab/alpaca_xhosa_taco": (62_000, 10_000_000),
+    "saillab/alpaca_chichewa_taco": (62_000, 10_000_000),
+    "EthioNLP/Amharic_Instruction_dataset": (154_000, 25_000_000),
+
+    # === MASSIVE WEB CRAWL NEW ===
+    "yhavinga/ccmatrix": (7_400_000_000, 7_400_000_000),
+    "sentence-transformers/parallel-sentences-ccmatrix": (1_000_000_000, 1_000_000_000),
+    "datalama/pretrain-nllb-filtered": (1_000_000_000, 1_000_000_000),
+    "hotchpotch/nllb-english-bitext-hq": (100_000_000, 15_000_000_000),
+    "CIRCE-lab/AFROXLMR": (421_000, 60_000_000),
+
+    # === AFRICAN MULTILINGUAL CORPORA ===
+    "NaolBM/african-corpus": (35_300_000, 5_000_000_000),
+    "bonadossou/afrolm_active_learning_dataset": (5_000_000, 750_000_000),
+    "csebuetnlp/xlsum": (1_350_000, 200_000_000),
+    "csebuetnlp/mt5_xlsum_all_languages_44": (863_000, 120_000_000),
+    "masakhane/african-translated-alpaca": (500_000, 75_000_000),
+    "masakhane/african-ultrachat": (55_000, 10_000_000),
+
+    # === HATE SPEECH ===
+    "masakhane/afri-hate-speech": (15_000, 2_000_000),
+
+    # === NLI / XNLI ===
+    "masakhane/afro-xnli": (392_000, 50_000_000),
+    "facebook/xnli": (392_000, 50_000_000),
+
+    # === POS TAGGING ===
+    "masakhane/masakhapos": (50_000, 2_000_000),
+
+    # === NEWS ===
+    "masakhane/masakhanews": (100_000, 15_000_000),
+    "community-datasets/swahili_news": (200_000, 30_000_000),
+
+    # === QA BENCHMARKS NEW ===
+    "google-research-datasets/tydiqa": (217_000, 40_000_000),
+    "cambridgeltl/xcopa": (6_600, 1_000_000),
+    "masakhane/jfleg_kin": (846, 100_000),
+
+    # === SENTIMENT NEW ===
+    "shmuhammad/AfriSenti-twitter-sentiment": (100_000, 10_000_000),
+    "masakhane/safri-sentiment-23": (50_000, 5_000_000),
+
+    # === MORE MICHSETHOWUSU PARALLEL ===
+    "michsethowusu/english-kinyarwanda_sentence-pairs": (200_000, 30_000_000),
+    "michsethowusu/portuguese-kinyarwanda_sentence-pairs": (150_000, 22_000_000),
+    "michsethowusu/german-kinyarwanda_sentence-pairs": (120_000, 18_000_000),
+    "michsethowusu/swahili-kinyarwanda_sentence-pairs": (180_000, 27_000_000),
+    "michsethowusu/hausa-kinyarwanda_sentence-pairs": (90_000, 13_000_000),
+    "michsethowusu/igbo-kinyarwanda_sentence-pairs": (80_000, 12_000_000),
+    "michsethowusu/luganda-kinyarwanda_sentence-pairs": (70_000, 10_000_000),
+    "michsethowusu/kinyarwanda-lingala_sentence-pairs": (60_000, 9_000_000),
+    "michsethowusu/kinyarwanda-kamba_sentence-pairs": (45_000, 6_500_000),
+    "michsethowusu/kinyarwanda-bambara_sentence-pairs": (55_000, 8_000_000),
+    "michsethowusu/kinyarwanda-krio_sentence-pairs": (35_000, 5_000_000),
+    "michsethowusu/kinyarwanda-twili_sentence-pairs": (40_000, 6_000_000),
+
+    # === CODE-170K VARIANTS ===
+    "michsethowusu/Code-170k-swahili": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-hausa": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-yoruba": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-amharic": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-zulu": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-xhosa": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-igbo": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-shona": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-afrikaans": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-luganda": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-bambara": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-lingala": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-tswana": (177_000, 80_000_000),
+
+    # === EMOTION CORPUS ===
+    "michsethowusu/emotion-corpus-kinyarwanda": (50_000, 5_000_000),
+    "michsethowusu/emotion-corpus-swahili": (50_000, 5_000_000),
+    "michsethowusu/emotion-corpus-hausa": (50_000, 5_000_000),
+    "michsethowusu/emotion-corpus-yoruba": (50_000, 5_000_000),
+    "michsethowusu/emotion-corpus-amharic": (50_000, 5_000_000),
+    "michsethowusu/emotion-corpus-zulu": (50_000, 5_000_000),
+    "michsethowusu/emotion-corpus-xhosa": (50_000, 5_000_000),
+
+    # === SPEECH-TEXT PARALLEL ===
+    "michsethowusu/speech-text-kinyarwanda": (100_000, 15_000_000),
+    "michsethowusu/speech-text-swahili": (100_000, 15_000_000),
+    "michsethowusu/speech-text-hausa": (100_000, 15_000_000),
+    "michsethowusu/speech-text-yoruba": (100_000, 15_000_000),
+    "michsethowusu/speech-text-igbo": (100_000, 15_000_000),
+    "michsethowusu/speech-text-amharic": (100_000, 15_000_000),
+    "michsethowusu/speech-text-zulu": (100_000, 15_000_000),
+
+    # === CROSS-LANGUAGE PARALLEL ===
+    "michsethowusu/swahili-zulu_sentence-pairs": (80_000, 12_000_000),
+    "michsethowusu/hausa-igbo_sentence-pairs": (70_000, 10_000_000),
+    "michsethowusu/yoruba-igbo_sentence-pairs": (65_000, 9_500_000),
+    "michsethowusu/amharic-igbo_sentence-pairs": (60_000, 9_000_000),
+    "michsethowusu/swahili-hausa_sentence-pairs": (75_000, 11_000_000),
+    "michsethowusu/zulu-xhosa_sentence-pairs": (55_000, 8_000_000),
+    "michsethowusu/shona-zulu_sentence-pairs": (50_000, 7_500_000),
+    "michsethowusu/luganda-swahili_sentence-pairs": (60_000, 9_000_000),
+    "michsethowusu/amharic-swahili_sentence-pairs": (65_000, 9_500_000),
+
+    # === WIKIPEDIA / WIKIDATA ===
+    "wikimedia/wikipedia": (5_000_000, 1_000_000_000),
+    "wikimedia/wikidata-title-desc": (88_300_000, 2_000_000_000),
+
+    # === FLORES / FLEURS ===
+    "facebook/flores": (2_000, 500_000),  # devtest
+    "google/fleurs": (100_000, 15_000_000),
+
+    # === ADDITIONAL KIN PAIRS ===
+    "michsethowusu/kinyarwanda-swahili_sentence-pairs": (160_000, 24_000_000),
+    "michsethowusu/kinyarwanda-hausa_sentence-pairs": (85_000, 12_500_000),
+    "michsethowusu/kinyarwanda-igbo_sentence-pairs": (75_000, 11_000_000),
+    "michsethowusu/kinyarwanda-amharic_sentence-pairs": (90_000, 13_000_000),
+    "michsethowusu/kinyarwanda-luganda_sentence-pairs": (70_000, 10_000_000),
+    "michsethowusu/kinyarwanda-bemba_sentence-pairs": (60_000, 9_000_000),
+    "michsethowusu/kinyarwanda-twili_sentence-pairs": (40_000, 6_000_000),
+    "michsethowusu/kinyarwanda-mossi_sentence-pairs": (35_000, 5_000_000),
+    "michsethowusu/kinyarwanda-ewondo_sentence-pairs": (30_000, 4_500_000),
+    "michsethowusu/kinyarwanda-bassa_sentence-pairs": (25_000, 3_500_000),
+    "michsethowusu/kinyarwanda-tigrinya_sentence-pairs": (40_000, 6_000_000),
+    "michsethowusu/kinyarwanda-fon_sentence-pairs": (20_000, 3_000_000),
+    "michsethowusu/kinyarwanda-ga_sentence-pairs": (25_000, 3_500_000),
+    "michsethowusu/kinyarwanda-wolof_sentence-pairs": (30_000, 4_500_000),
+    "michsethowusu/kinyarwanda-mandinka_sentence-pairs": (25_000, 3_500_000),
+    "michsethowusu/kinyarwanda-serer_sentence-pairs": (20_000, 3_000_000),
+    "michsethowusu/kinyarwanda-kinyarwanda_nllb_sentence-pairs": (100_000, 15_000_000),
+    "michsethowusu/afro-xlmr-kinyarwanda": (50_000, 7_500_000),
+
+    # === ADDITIONAL MICHSETHOWUSU CROSS-PARALLEL ===
+    "michsethowusu/french-swahili_sentence-pairs": (300_000, 45_000_000),
+    "michsethowusu/french-hausa_sentence-pairs": (200_000, 30_000_000),
+    "michsethowusu/french-yoruba_sentence-pairs": (180_000, 27_000_000),
+    "michsethowusu/french-igbo_sentence-pairs": (150_000, 22_000_000),
+    "michsethowusu/french-zulu_sentence-pairs": (120_000, 18_000_000),
+    "michsethowusu/french-amharic_sentence-pairs": (140_000, 21_000_000),
+    "michsethowusu/english-swahili_sentence-pairs": (400_000, 60_000_000),
+    "michsethowusu/english-hausa_sentence-pairs": (300_000, 45_000_000),
+    "michsethowusu/english-yoruba_sentence-pairs": (250_000, 37_000_000),
+    "michsethowusu/english-amharic_sentence-pairs": (200_000, 30_000_000),
+    "michsethowusu/english-zulu_sentence-pairs": (180_000, 27_000_000),
+    "michsethowusu/english-shona_sentence-pairs": (150_000, 22_000_000),
+    "michsethowusu/english-igbo_sentence-pairs": (200_000, 30_000_000),
+    "michsethowusu/english-xhosa_sentence-pairs": (120_000, 18_000_000),
+    "michsethowusu/english-luganda_sentence-pairs": (100_000, 15_000_000),
+    "michsethowusu/german-swahili_sentence-pairs": (80_000, 12_000_000),
+    "michsethowusu/german-hausa_sentence-pairs": (60_000, 9_000_000),
+    "michsethowusu/spanish-swahili_sentence-pairs": (70_000, 10_500_000),
+    "michsethowusu/spanish-hausa_sentence-pairs": (50_000, 7_500_000),
+
+    # === UHURA BENCHMARKS ===
+    "masakhane/mafand": (1_471, 300_000),
+    "masakhane/uhura-hellaswag": (10_000, 2_000_000),
+    "masakhane/uhura-winogrande": (5_000, 1_000_000),
+    "masakhane/uhura-sciq": (8_000, 1_500_000),
+    "masakhane/uhura-openbookqa": (5_000, 1_000_000),
+    "masakhane/uhura-mmlu": (15_000, 3_000_000),
+    "masakhane/uhura-gsm8k": (8_000, 1_500_000),
+    "masakhane/uhura-arctest": (10_000, 2_000_000),
+
+    # === LUGHA GEN ===
+    "LughaGen-Organization/LughaGen-Kinyarwanda": (100_000, 15_000_000),
+    "LughaGen-Organization/LughaGen-Swahili": (100_000, 15_000_000),
+    "LughaGen-Organization/LughaGen-Hausa": (100_000, 15_000_000),
+    "LughaGen-Organization/LughaGen-Yoruba": (100_000, 15_000_000),
+    "LughaGen-Organization/LughaGen-Amharic": (100_000, 15_000_000),
+    "LughaGen-Organization/LughaGen-Zulu": (100_000, 15_000_000),
+    "LughaGen-Organization/LughaGen-Igbo": (100_000, 15_000_000),
+    "LughaGen-Organization/LughaGen-Shona": (100_000, 15_000_000),
+
+    # === FINAL 4 ===
+    "michsethowusu/kinyarwanda-sango_sentence-pairs": (30_000, 4_500_000),
+    "michsethowusu/kinyarwanda-fulfulde_sentence-pairs": (35_000, 5_000_000),
+    "michsethowusu/Code-170k-amharic_swahili": (177_000, 80_000_000),
+    "michsethowusu/Code-170k-luganda_kinyarwanda": (177_000, 80_000_000),
+}
+
+total_datasets = len(datasets)
+total_rows = sum(r for r, t in dataset_stats.values())
+total_tokens = sum(t for r, t in dataset_stats.values())
+
+print("=" * 70)
+print("BWENGE AI - COMPLETE STATISTICS")
+print("=" * 70)
+print(f"Total Datasets:    {total_datasets}")
+print(f"Total Rows:        {total_rows:>20,}")
+print(f"Total Tokens:      {total_tokens:>20,}")
+print(f"                    ({total_tokens / 1_000_000_000:.1f}B tokens)")
+print("=" * 70)
+
+# Breakdown by category
+categories = {
+    "Monolingual Kinyarwanda": ["CircuitNotion/", "mbazaNLP/kinyarwanda_monolingual"],
+    "Massive Web Crawl": ["HuggingFaceFW/", "HPLT/", "yhavinga/", "datalama/", "hotchpotch/"],
+    "Parallel Corpus": ["michsethowusu/"],
+    "Helsinki-NLP": ["Helsinki-NLP/"],
+    "AllenAI NLLB": ["allenai/"],
+    "CCAligned": ["ahelk/"],
+    "Instruction Tuning": ["saillab/", "ChrisToukmaji/", "EthioNLP/"],
+    "Chat / Conversation": ["Mikecyane/", "jojo-ai-mst/"],
+    "Wikipedia": ["RogerB/", "wikimedia/"],
+    "NER / POS": ["masakhane/masakhaner", "masakhane/masakhapos"],
+    "QA Benchmarks": ["masakhane/afriqa", "masakhane/uhura", "masakhane/jfleg", "google-research/", "cambridgeltl/", "mbazaNLP/PIQA"],
+    "Sentiment": ["masakhane/afrisenti", "masakhane/safri", "shmuhammad/"],
+    "NLI": ["masakhane/afrixnli", "masakhane/afro-xnli", "facebook/xnli"],
+    "MMLU": ["masakhane/afrimmlu"],
+    "Intent": ["masakhane/InjongoIntent"],
+    "Math": ["masakhane/afrimgsm", "masakhane/uhura-gsm"],
+    "Dialogue": ["masakhane/AfriADR"],
+    "Translation MT": ["facebook/flores", "openlanguagedata/", "masakhane/mafand", "google/fleurs"],
+    "Health": ["DigitalUmuganda/Monolingual"],
+    "Common Voice": ["mbazaNLP/common-voice", "Kira-Floris/"],
+    "African Multilingual": ["NaolBM/", "bonadossou/", "csebuetnlp/", "LughaGen/", "CIRCE-lab/", "community-datasets/"],
+    "Code": ["michsethowusu/Code-"],
+    "Emotion": ["michsethowusu/emotion-corpus"],
+    "Speech-Text": ["michsethowusu/speech-text"],
+    "Hate Speech": ["masakhane/afri-hate"],
+}
+
+print("\nBREAKDOWN BY CATEGORY:")
+print("-" * 70)
+for cat_name, prefixes in categories.items():
+    cat_rows = 0
+    cat_tokens = 0
+    cat_count = 0
+    for ds in datasets:
+        name = ds["name"]
+        if any(name.startswith(p) or name == p for p in prefixes):
+            r, t = dataset_stats.get(name, (0, 0))
+            cat_rows += r
+            cat_tokens += t
+            cat_count += 1
+    if cat_count > 0:
+        print(f"  {cat_name:<30} {cat_count:>3} datasets  {cat_rows:>15,} rows  {cat_tokens / 1e9:>8.1f}B tokens")
+
+print("-" * 70)
+print(f"  {'TOTAL':<30} {total_datasets:>3} datasets  {total_rows:>15,} rows  {total_tokens / 1e9:>8.1f}B tokens")
+print("=" * 70)
+
+# Top 10 by token volume
+print("\nTOP 10 DATASETS BY TOKEN VOLUME:")
+print("-" * 70)
+sorted_ds = sorted(dataset_stats.items(), key=lambda x: x[1][1], reverse=True)
+for i, (name, (rows, tokens)) in enumerate(sorted_ds[:10], 1):
+    print(f"  {i:>2}. {name}")
+    print(f"      {rows:>15,} rows  |  {tokens / 1e9:>8.1f}B tokens")
+print("=" * 70)
+
+# Status vs targets
+print("\nTARGET PROGRESS:")
+print("-" * 70)
+print(f"  Datasets: {total_datasets}/200    ({total_datasets/200*100:.0f}%)")
+print(f"  Rows:     {total_rows/1e9:.2f}B/1B    ({total_rows/1e9*100:.0f}%)")
+print(f"  Tokens:   {total_tokens/1e9:.1f}B/70B   ({total_tokens/70e9*100:.1f}%)")
+print("=" * 70)