Upload Base/scripts/build_english_corpus.py with huggingface_hub

Base/scripts/build_english_corpus.py

@@ -1,349 +1,349 @@
-"""
-Build an ultra-clean English text corpus for continued pretraining.
-
-Downloads from high-quality, professionally curated sources via HuggingFace
-streaming (constant memory, no full dataset download needed):
-
-  1. Wikipedia English — encyclopedic, multi-editor-reviewed text
-     (the single cleanest large-scale English corpus in existence)
-  2. FineWeb-Edu — top-scored educational web content (score >= 4.0)
-     (adds diversity: textbooks, tutorials, explanations, articles)
-
-Quality filters applied to every text:
-  - Minimum length and word count
-  - High alphabetic ratio (rejects tables, code, symbol dumps)
-  - Sentence structure check (real prose, not lists/fragments)
-  - Repetition detection (rejects copy-paste / boilerplate)
-  - URL density filter (rejects link farms)
-  - Unicode normalization and whitespace cleanup
-
-Outputs parquet files to Base/data/filtered_english/ with a single 'text'
-column, compatible with the existing prepare_litdata.py pipeline.
-
-After running this script:
-  1. python Base/scripts/prepare_litdata.py --mode english
-  2. litgpt pretrain --config Base/configs/pretrain_100m_english.yaml
-
-Usage:
-  python Base/scripts/build_english_corpus.py
-  python Base/scripts/build_english_corpus.py --target_tokens 100000000
-  python Base/scripts/build_english_corpus.py --sources wiki
-  python Base/scripts/build_english_corpus.py --sources fineweb
-  python Base/scripts/build_english_corpus.py --fineweb_min_score 4.5
-"""
-
-import argparse
-import os
-import re
-import time
-import unicodedata
-from pathlib import Path
-
-import pyarrow as pa
-import pyarrow.parquet as pq
-
-
-# ─── Text Quality Filters ──────────────────────────────────────────────────
-
-def clean_text(text: str) -> str:
-    """Normalize Unicode, strip control chars, fix whitespace."""
-    # Normalize unicode (NFKC merges compatibility chars)
-    text = unicodedata.normalize("NFKC", text)
-    # Remove control characters (keep newlines and tabs)
-    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
-    # Collapse 3+ blank lines into 2
-    text = re.sub(r'\n{3,}', '\n\n', text)
-    # Collapse multiple spaces/tabs into single space
-    text = re.sub(r'[ \t]+', ' ', text)
-    # Strip each line
-    text = '\n'.join(line.strip() for line in text.split('\n'))
-    return text.strip()
-
-
-def is_high_quality(text: str, min_chars: int = 500, min_words: int = 80) -> bool:
-    """Strict quality gate — only passes clean, well-formed English prose."""
-    if len(text) < min_chars:
-        return False
-
-    words = text.split()
-    num_words = len(words)
-    if num_words < min_words:
-        return False
-
-    # Must be mostly alphabetic (not tables, code, numbers)
-    alpha = sum(c.isalpha() for c in text)
-    if alpha / max(len(text), 1) < 0.65:
-        return False
-
-    # Average word length sanity check (2.5–15 chars for English)
-    avg_word_len = sum(len(w) for w in words) / num_words
-    if avg_word_len < 2.5 or avg_word_len > 15:
-        return False
-
-    # Not too many URLs (< 3% of words)
-    url_hits = text.count('http://') + text.count('https://')
-    if url_hits > num_words * 0.03:
-        return False
-
-    # Must contain real sentences (at least 3 sentences > 10 chars)
-    sentences = re.split(r'[.!?]+', text)
-    real_sentences = [s.strip() for s in sentences if len(s.strip()) > 10]
-    if len(real_sentences) < 3:
-        return False
-
-    # Repetition filter — unique lines should be > 50%
-    lines = [ln.strip() for ln in text.split('\n') if ln.strip()]
-    if len(lines) > 5:
-        unique_ratio = len(set(lines)) / len(lines)
-        if unique_ratio < 0.5:
-            return False
-
-    return True
-
-
-# ─── Wikipedia Source ───────────────────────────────────────────────────────
-
-# Wikipedia articles with these title patterns are low-value for language model
-_WIKI_SKIP_PATTERNS = re.compile(
-    r'(disambiguation|list of|lists of|index of|outline of|'
-    r'wikipedia:|template:|category:|portal:|module:|mediawiki:)',
-    re.IGNORECASE
-)
-
-
-def fetch_wikipedia(target_tokens: int, output_dir: str, tokens_per_word: float = 1.3):
-    """Stream Wikipedia English, apply strict quality filters, output parquet."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[Wikipedia] Streaming English articles")
-    print(f"  Target: {target_tokens:,} tokens")
-    print(f"  Output: {output_dir}")
-    print(f"{'='*60}\n")
-
-    os.makedirs(output_dir, exist_ok=True)
-
-    ds = load_dataset(
-        "wikimedia/wikipedia", "20231101.en",
-        split="train", streaming=True,
-        trust_remote_code=False
-    )
-
-    buf = []
-    total_tokens = 0
-    kept = 0
-    skipped = 0
-    file_idx = 0
-    BATCH = 5000
-    t0 = time.time()
-
-    for article in ds:
-        title = (article.get("title") or "").strip()
-        raw = article.get("text") or ""
-
-        # Skip meta / navigation articles
-        if _WIKI_SKIP_PATTERNS.search(title):
-            skipped += 1
-            continue
-
-        cleaned = clean_text(raw)
-
-        # Strict filter: min 800 chars, 120 words for Wikipedia
-        if not is_high_quality(cleaned, min_chars=800, min_words=120):
-            skipped += 1
-            continue
-
-        # Prepend title as natural heading
-        full_text = f"{title}\n\n{cleaned}"
-        est_tok = int(len(full_text.split()) * tokens_per_word)
-
-        buf.append(full_text)
-        total_tokens += est_tok
-        kept += 1
-
-        # Flush batch to parquet
-        if len(buf) >= BATCH:
-            fp = Path(output_dir) / f"wiki_{file_idx:04d}.parquet"
-            pq.write_table(pa.table({"text": buf}), str(fp))
-            elapsed = time.time() - t0
-            rate = total_tokens / max(elapsed, 1)
-            print(f"  wiki_{file_idx:04d}.parquet | {kept:,} articles | "
-                  f"~{total_tokens:,} tok | {rate:,.0f} tok/s | {elapsed:.0f}s")
-            buf = []
-            file_idx += 1
-
-        if total_tokens >= target_tokens:
-            break
-
-    # Write remaining
-    if buf:
-        fp = Path(output_dir) / f"wiki_{file_idx:04d}.parquet"
-        pq.write_table(pa.table({"text": buf}), str(fp))
-        file_idx += 1
-
-    elapsed = time.time() - t0
-    print(f"\n--- Wikipedia complete ---")
-    print(f"  Kept: {kept:,} articles | Skipped: {skipped:,}")
-    print(f"  Est. tokens: {total_tokens:,}")
-    print(f"  Files: {file_idx} | Time: {elapsed:.0f}s")
-    return total_tokens
-
-
-# ─── FineWeb-Edu Source ─────────────────────────────────────────────────────
-
-def fetch_fineweb_edu(target_tokens: int, output_dir: str,
-                      min_score: float = 4.0, tokens_per_word: float = 1.3):
-    """Stream FineWeb-Edu (top educational web content), output parquet."""
-    from datasets import load_dataset
-
-    print(f"\n{'='*60}")
-    print(f"[FineWeb-Edu] Streaming (score >= {min_score})")
-    print(f"  Target: {target_tokens:,} tokens")
-    print(f"  Output: {output_dir}")
-    print(f"{'='*60}\n")
-
-    os.makedirs(output_dir, exist_ok=True)
-
-    ds = load_dataset(
-        "HuggingFaceFW/fineweb-edu", "sample-10BT",
-        split="train", streaming=True,
-        trust_remote_code=False
-    )
-
-    buf = []
-    total_tokens = 0
-    kept = 0
-    skipped = 0
-    file_idx = 0
-    BATCH = 5000
-    t0 = time.time()
-
-    for doc in ds:
-        # FineWeb-Edu has 'score' (float) — educational quality 0-5
-        score = doc.get("score", 0)
-        if not isinstance(score, (int, float)):
-            try:
-                score = float(score)
-            except (ValueError, TypeError):
-                skipped += 1
-                continue
-
-        if score < min_score:
-            skipped += 1
-            continue
-
-        raw = doc.get("text") or ""
-        cleaned = clean_text(raw)
-
-        if not is_high_quality(cleaned, min_chars=500, min_words=80):
-            skipped += 1
-            continue
-
-        est_tok = int(len(cleaned.split()) * tokens_per_word)
-
-        buf.append(cleaned)
-        total_tokens += est_tok
-        kept += 1
-
-        if len(buf) >= BATCH:
-            fp = Path(output_dir) / f"fineweb_{file_idx:04d}.parquet"
-            pq.write_table(pa.table({"text": buf}), str(fp))
-            elapsed = time.time() - t0
-            rate = total_tokens / max(elapsed, 1)
-            print(f"  fineweb_{file_idx:04d}.parquet | {kept:,} docs | "
-                  f"~{total_tokens:,} tok | {rate:,.0f} tok/s | {elapsed:.0f}s")
-            buf = []
-            file_idx += 1
-
-        if total_tokens >= target_tokens:
-            break
-
-    if buf:
-        fp = Path(output_dir) / f"fineweb_{file_idx:04d}.parquet"
-        pq.write_table(pa.table({"text": buf}), str(fp))
-        file_idx += 1
-
-    elapsed = time.time() - t0
-    print(f"\n--- FineWeb-Edu complete ---")
-    print(f"  Kept: {kept:,} docs | Skipped: {skipped:,}")
-    print(f"  Est. tokens: {total_tokens:,}")
-    print(f"  Files: {file_idx} | Time: {elapsed:.0f}s")
-    return total_tokens
-
-
-# ─── Main ───────────────────────────────────────────────────────────────────
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Build ultra-clean English corpus for continued pretraining"
-    )
-    parser.add_argument(
-        "--target_tokens", type=int, default=50_000_000,
-        help="Total target token count (default: 50,000,000 = 50M)"
-    )
-    parser.add_argument(
-        "--sources", type=str, default="wiki,fineweb",
-        help="Comma-separated sources: wiki, fineweb (default: wiki,fineweb)"
-    )
-    parser.add_argument(
-        "--wiki_share", type=float, default=0.6,
-        help="Wikipedia share when both sources are used (default: 0.6 = 60%%)"
-    )
-    parser.add_argument(
-        "--fineweb_min_score", type=float, default=4.0,
-        help="Minimum FineWeb-Edu educational score 0-5 (default: 4.0)"
-    )
-    parser.add_argument(
-        "--output_dir", type=str, default="Base/data/filtered_english",
-        help="Output directory for parquet files"
-    )
-    args = parser.parse_args()
-
-    sources = [s.strip().lower() for s in args.sources.split(",")]
-
-    print(f"\n{'#'*60}")
-    print(f"  ULTRA-CLEAN ENGLISH CORPUS BUILDER")
-    print(f"  Target:  {args.target_tokens:,} tokens")
-    print(f"  Sources: {', '.join(sources)}")
-    print(f"  Output:  {args.output_dir}")
-    print(f"{'#'*60}")
-
-    # Check datasets library
-    try:
-        import datasets
-        print(f"  datasets v{datasets.__version__}")
-    except ImportError:
-        print("\n  ERROR: 'datasets' library is required for streaming.")
-        print("  Install it:  pip install datasets")
-        return
-
-    total = 0
-
-    if "wiki" in sources:
-        if "fineweb" in sources:
-            wiki_target = int(args.target_tokens * args.wiki_share)
-        else:
-            wiki_target = args.target_tokens
-        total += fetch_wikipedia(wiki_target, args.output_dir)
-
-    if "fineweb" in sources:
-        fw_target = args.target_tokens - total if total > 0 else args.target_tokens
-        if fw_target > 0:
-            total += fetch_fineweb_edu(
-                fw_target, args.output_dir,
-                min_score=args.fineweb_min_score
-            )
-
-    print(f"\n{'#'*60}")
-    print(f"  CORPUS BUILD COMPLETE")
-    print(f"  Total estimated tokens: {total:,}")
-    print(f"  Output directory: {args.output_dir}")
-    print(f"{'#'*60}")
-    print(f"\nNext steps:")
-    print(f"  1. python Base/scripts/prepare_litdata.py --mode english")
-    print(f"  2. $env:TORCHDYNAMO_DISABLE = '1'")
-    print(f"  3. litgpt pretrain --config Base/configs/pretrain_100m_english.yaml")
-
-
-if __name__ == "__main__":
-    main()
+"""
+Build an ultra-clean English text corpus for continued pretraining.
+
+Downloads from high-quality, professionally curated sources via HuggingFace
+streaming (constant memory, no full dataset download needed):
+
+  1. Wikipedia English — encyclopedic, multi-editor-reviewed text
+     (the single cleanest large-scale English corpus in existence)
+  2. FineWeb-Edu — top-scored educational web content (score >= 4.0)
+     (adds diversity: textbooks, tutorials, explanations, articles)
+
+Quality filters applied to every text:
+  - Minimum length and word count
+  - High alphabetic ratio (rejects tables, code, symbol dumps)
+  - Sentence structure check (real prose, not lists/fragments)
+  - Repetition detection (rejects copy-paste / boilerplate)
+  - URL density filter (rejects link farms)
+  - Unicode normalization and whitespace cleanup
+
+Outputs parquet files to Base/data/filtered_english/ with a single 'text'
+column, compatible with the existing prepare_litdata.py pipeline.
+
+After running this script:
+  1. python Base/scripts/prepare_litdata.py --mode english
+  2. litgpt pretrain --config Base/configs/pretrain_100m_english.yaml
+
+Usage:
+  python Base/scripts/build_english_corpus.py
+  python Base/scripts/build_english_corpus.py --target_tokens 100000000
+  python Base/scripts/build_english_corpus.py --sources wiki
+  python Base/scripts/build_english_corpus.py --sources fineweb
+  python Base/scripts/build_english_corpus.py --fineweb_min_score 4.5
+"""
+
+import argparse
+import os
+import re
+import time
+import unicodedata
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+# ─── Text Quality Filters ──────────────────────────────────────────────────
+
+def clean_text(text: str) -> str:
+    """Normalize Unicode, strip control chars, fix whitespace."""
+    # Normalize unicode (NFKC merges compatibility chars)
+    text = unicodedata.normalize("NFKC", text)
+    # Remove control characters (keep newlines and tabs)
+    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]', '', text)
+    # Collapse 3+ blank lines into 2
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    # Collapse multiple spaces/tabs into single space
+    text = re.sub(r'[ \t]+', ' ', text)
+    # Strip each line
+    text = '\n'.join(line.strip() for line in text.split('\n'))
+    return text.strip()
+
+
+def is_high_quality(text: str, min_chars: int = 500, min_words: int = 80) -> bool:
+    """Strict quality gate — only passes clean, well-formed English prose."""
+    if len(text) < min_chars:
+        return False
+
+    words = text.split()
+    num_words = len(words)
+    if num_words < min_words:
+        return False
+
+    # Must be mostly alphabetic (not tables, code, numbers)
+    alpha = sum(c.isalpha() for c in text)
+    if alpha / max(len(text), 1) < 0.65:
+        return False
+
+    # Average word length sanity check (2.5–15 chars for English)
+    avg_word_len = sum(len(w) for w in words) / num_words
+    if avg_word_len < 2.5 or avg_word_len > 15:
+        return False
+
+    # Not too many URLs (< 3% of words)
+    url_hits = text.count('http://') + text.count('https://')
+    if url_hits > num_words * 0.03:
+        return False
+
+    # Must contain real sentences (at least 3 sentences > 10 chars)
+    sentences = re.split(r'[.!?]+', text)
+    real_sentences = [s.strip() for s in sentences if len(s.strip()) > 10]
+    if len(real_sentences) < 3:
+        return False
+
+    # Repetition filter — unique lines should be > 50%
+    lines = [ln.strip() for ln in text.split('\n') if ln.strip()]
+    if len(lines) > 5:
+        unique_ratio = len(set(lines)) / len(lines)
+        if unique_ratio < 0.5:
+            return False
+
+    return True
+
+
+# ─── Wikipedia Source ───────────────────────────────────────────────────────
+
+# Wikipedia articles with these title patterns are low-value for language model
+_WIKI_SKIP_PATTERNS = re.compile(
+    r'(disambiguation|list of|lists of|index of|outline of|'
+    r'wikipedia:|template:|category:|portal:|module:|mediawiki:)',
+    re.IGNORECASE
+)
+
+
+def fetch_wikipedia(target_tokens: int, output_dir: str, tokens_per_word: float = 1.3):
+    """Stream Wikipedia English, apply strict quality filters, output parquet."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[Wikipedia] Streaming English articles")
+    print(f"  Target: {target_tokens:,} tokens")
+    print(f"  Output: {output_dir}")
+    print(f"{'='*60}\n")
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    ds = load_dataset(
+        "wikimedia/wikipedia", "20231101.en",
+        split="train", streaming=True,
+        trust_remote_code=False
+    )
+
+    buf = []
+    total_tokens = 0
+    kept = 0
+    skipped = 0
+    file_idx = 0
+    BATCH = 5000
+    t0 = time.time()
+
+    for article in ds:
+        title = (article.get("title") or "").strip()
+        raw = article.get("text") or ""
+
+        # Skip meta / navigation articles
+        if _WIKI_SKIP_PATTERNS.search(title):
+            skipped += 1
+            continue
+
+        cleaned = clean_text(raw)
+
+        # Strict filter: min 800 chars, 120 words for Wikipedia
+        if not is_high_quality(cleaned, min_chars=800, min_words=120):
+            skipped += 1
+            continue
+
+        # Prepend title as natural heading
+        full_text = f"{title}\n\n{cleaned}"
+        est_tok = int(len(full_text.split()) * tokens_per_word)
+
+        buf.append(full_text)
+        total_tokens += est_tok
+        kept += 1
+
+        # Flush batch to parquet
+        if len(buf) >= BATCH:
+            fp = Path(output_dir) / f"wiki_{file_idx:04d}.parquet"
+            pq.write_table(pa.table({"text": buf}), str(fp))
+            elapsed = time.time() - t0
+            rate = total_tokens / max(elapsed, 1)
+            print(f"  wiki_{file_idx:04d}.parquet | {kept:,} articles | "
+                  f"~{total_tokens:,} tok | {rate:,.0f} tok/s | {elapsed:.0f}s")
+            buf = []
+            file_idx += 1
+
+        if total_tokens >= target_tokens:
+            break
+
+    # Write remaining
+    if buf:
+        fp = Path(output_dir) / f"wiki_{file_idx:04d}.parquet"
+        pq.write_table(pa.table({"text": buf}), str(fp))
+        file_idx += 1
+
+    elapsed = time.time() - t0
+    print(f"\n--- Wikipedia complete ---")
+    print(f"  Kept: {kept:,} articles | Skipped: {skipped:,}")
+    print(f"  Est. tokens: {total_tokens:,}")
+    print(f"  Files: {file_idx} | Time: {elapsed:.0f}s")
+    return total_tokens
+
+
+# ─── FineWeb-Edu Source ─────────────────────────────────────────────────────
+
+def fetch_fineweb_edu(target_tokens: int, output_dir: str,
+                      min_score: float = 4.0, tokens_per_word: float = 1.3):
+    """Stream FineWeb-Edu (top educational web content), output parquet."""
+    from datasets import load_dataset
+
+    print(f"\n{'='*60}")
+    print(f"[FineWeb-Edu] Streaming (score >= {min_score})")
+    print(f"  Target: {target_tokens:,} tokens")
+    print(f"  Output: {output_dir}")
+    print(f"{'='*60}\n")
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    ds = load_dataset(
+        "HuggingFaceFW/fineweb-edu", "sample-10BT",
+        split="train", streaming=True,
+        trust_remote_code=False
+    )
+
+    buf = []
+    total_tokens = 0
+    kept = 0
+    skipped = 0
+    file_idx = 0
+    BATCH = 5000
+    t0 = time.time()
+
+    for doc in ds:
+        # FineWeb-Edu has 'score' (float) — educational quality 0-5
+        score = doc.get("score", 0)
+        if not isinstance(score, (int, float)):
+            try:
+                score = float(score)
+            except (ValueError, TypeError):
+                skipped += 1
+                continue
+
+        if score < min_score:
+            skipped += 1
+            continue
+
+        raw = doc.get("text") or ""
+        cleaned = clean_text(raw)
+
+        if not is_high_quality(cleaned, min_chars=500, min_words=80):
+            skipped += 1
+            continue
+
+        est_tok = int(len(cleaned.split()) * tokens_per_word)
+
+        buf.append(cleaned)
+        total_tokens += est_tok
+        kept += 1
+
+        if len(buf) >= BATCH:
+            fp = Path(output_dir) / f"fineweb_{file_idx:04d}.parquet"
+            pq.write_table(pa.table({"text": buf}), str(fp))
+            elapsed = time.time() - t0
+            rate = total_tokens / max(elapsed, 1)
+            print(f"  fineweb_{file_idx:04d}.parquet | {kept:,} docs | "
+                  f"~{total_tokens:,} tok | {rate:,.0f} tok/s | {elapsed:.0f}s")
+            buf = []
+            file_idx += 1
+
+        if total_tokens >= target_tokens:
+            break
+
+    if buf:
+        fp = Path(output_dir) / f"fineweb_{file_idx:04d}.parquet"
+        pq.write_table(pa.table({"text": buf}), str(fp))
+        file_idx += 1
+
+    elapsed = time.time() - t0
+    print(f"\n--- FineWeb-Edu complete ---")
+    print(f"  Kept: {kept:,} docs | Skipped: {skipped:,}")
+    print(f"  Est. tokens: {total_tokens:,}")
+    print(f"  Files: {file_idx} | Time: {elapsed:.0f}s")
+    return total_tokens
+
+
+# ─── Main ───────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Build ultra-clean English corpus for continued pretraining"
+    )
+    parser.add_argument(
+        "--target_tokens", type=int, default=50_000_000,
+        help="Total target token count (default: 50,000,000 = 50M)"
+    )
+    parser.add_argument(
+        "--sources", type=str, default="wiki,fineweb",
+        help="Comma-separated sources: wiki, fineweb (default: wiki,fineweb)"
+    )
+    parser.add_argument(
+        "--wiki_share", type=float, default=0.6,
+        help="Wikipedia share when both sources are used (default: 0.6 = 60%%)"
+    )
+    parser.add_argument(
+        "--fineweb_min_score", type=float, default=4.0,
+        help="Minimum FineWeb-Edu educational score 0-5 (default: 4.0)"
+    )
+    parser.add_argument(
+        "--output_dir", type=str, default="Base/data/filtered_english",
+        help="Output directory for parquet files"
+    )
+    args = parser.parse_args()
+
+    sources = [s.strip().lower() for s in args.sources.split(",")]
+
+    print(f"\n{'#'*60}")
+    print(f"  ULTRA-CLEAN ENGLISH CORPUS BUILDER")
+    print(f"  Target:  {args.target_tokens:,} tokens")
+    print(f"  Sources: {', '.join(sources)}")
+    print(f"  Output:  {args.output_dir}")
+    print(f"{'#'*60}")
+
+    # Check datasets library
+    try:
+        import datasets
+        print(f"  datasets v{datasets.__version__}")
+    except ImportError:
+        print("\n  ERROR: 'datasets' library is required for streaming.")
+        print("  Install it:  pip install datasets")
+        return
+
+    total = 0
+
+    if "wiki" in sources:
+        if "fineweb" in sources:
+            wiki_target = int(args.target_tokens * args.wiki_share)
+        else:
+            wiki_target = args.target_tokens
+        total += fetch_wikipedia(wiki_target, args.output_dir)
+
+    if "fineweb" in sources:
+        fw_target = args.target_tokens - total if total > 0 else args.target_tokens
+        if fw_target > 0:
+            total += fetch_fineweb_edu(
+                fw_target, args.output_dir,
+                min_score=args.fineweb_min_score
+            )
+
+    print(f"\n{'#'*60}")
+    print(f"  CORPUS BUILD COMPLETE")
+    print(f"  Total estimated tokens: {total:,}")
+    print(f"  Output directory: {args.output_dir}")
+    print(f"{'#'*60}")
+    print(f"\nNext steps:")
+    print(f"  1. python Base/scripts/prepare_litdata.py --mode english")
+    print(f"  2. $env:TORCHDYNAMO_DISABLE = '1'")
+    print(f"  3. litgpt pretrain --config Base/configs/pretrain_100m_english.yaml")
+
+
+if __name__ == "__main__":
+    main()