Upload folder using huggingface_hub

README.md

@@ -1,13 +1,62 @@
 ---
-title: Cc Domain Graph Builder
-emoji: 😻
+title: CC Domain Backlink Graph Builder
+emoji: 🕸️
 colorFrom: indigo
-colorTo: green
+colorTo: gray
 sdk: gradio
-sdk_version: 6.15.2
-python_version: '3.13'
+sdk_version: 4.44.0
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Common Crawl domain backlink-graph builder
+
+A one-time (per quarterly release) build job that turns the Common Crawl
+**domain hyperlink graph** into partitioned Parquet on a Hugging Face dataset
+repo, so you can query *backlinks* ("who links to domain X") remotely with
+DuckDB — no 24 GB local download.
+
+## How it works
+
+`build.py` streams CC's gzipped graph CSVs straight from the CDN
+(`data.commoncrawl.org`) via DuckDB + httpfs, in a single pass:
+
+- **vertices** → `vertices_by_domain.parquet` (sorted by domain) +
+  `vertices_by_id.parquet` (sorted by id)
+- **ranks** → `ranks_by_id.parquet` (harmonic-centrality position per node)
+- **edges** → `edges/part=<n>/` partitioned by `to_id // part_size`
+  (default 1,000,000 → ~122 partitions)
+
+Because edges are bucketed by `to_id`, a backlink query reads only the one
+partition whose range contains the target's id — a few MB over HTTP Range,
+not the whole 22.9 GB edge set.
+
+## Deploy as a Space
+
+1. Create a new **Gradio Space** and add these files (`app.py`, `build.py`,
+   `requirements.txt`, this `README.md`).
+2. Add two **Space secrets**:
+   - `HF_TOKEN` — a write token for your target dataset repo
+   - `REPO` — e.g. `yourname/cc-domain-graph`
+3. Open the Space, click **Build & push**. Progress streams in the box.
+
+Notes:
+- The Space needs ~30 GB ephemeral disk for the Parquet output (the *source*
+  is streamed, not stored). If the build runs out of disk, upgrade the Space
+  hardware for the one-time run, or run `build.py` on Colab/Kaggle instead.
+- The build takes roughly 1–2 hours (dominated by streaming + partitioning the
+  22.9 GB edge file).
+
+## Or run it anywhere
+
+```bash
+pip install -r requirements.txt
+HF_TOKEN=hf_xxx python3 build.py --repo yourname/cc-domain-graph
+# or build locally without pushing:
+python3 build.py --repo x --no-push --out ./graph_out
+```
+
+## Refresh
+
+Common Crawl publishes a new graph quarterly. Bump `RELEASE` in `build.py`
+to the newest `cc-main-YYYY-...` and re-run to keep link rot bounded to months.

app.py

@@ -0,0 +1,76 @@
+"""Hugging Face Space: one-click build of the CC domain backlink graph.
+
+Set two Space secrets:
+  HF_TOKEN  — a write token for the target dataset repo
+  REPO      — target dataset repo id, e.g. "isam/cc-domain-graph"
+
+Then open the Space and click "Build & push". Progress streams below.
+The build streams the source from CC (no large source download to the Space);
+it writes ~25-30 GB of Parquet locally, so give the Space enough ephemeral
+disk (free tier is usually fine; upgrade hardware if it runs out of space).
+"""
+
+import os
+from pathlib import Path
+
+import gradio as gr
+
+from build import build, push, RELEASE
+
+
+def run(repo: str):
+    repo = (repo or os.environ.get("REPO") or "").strip()
+    token = os.environ.get("HF_TOKEN")
+    logs = []
+
+    def emit(line):
+        logs.append(line)
+        return "\n".join(logs)
+
+    if not repo:
+        yield emit("ERROR: set REPO secret or type a repo id (e.g. you/cc-domain-graph)")
+        return
+    if not token:
+        yield emit("ERROR: set the HF_TOKEN Space secret (write access).")
+        return
+
+    yield emit(f"Building {RELEASE} → {repo}")
+    out = Path("/tmp/graph_out")
+
+    # build() prints with flush; capture by redirecting print via a simple shim.
+    import builtins
+    orig_print = builtins.print
+    buffer = {"last": ""}
+
+    def teed_print(*a, **k):
+        msg = " ".join(str(x) for x in a)
+        buffer["last"] = msg
+        orig_print(*a, **k)
+
+    builtins.print = teed_print
+    try:
+        manifest = build(out)
+        yield emit(f"Built {manifest['domain_nodes']:,} nodes. Uploading ...")
+        push(out, repo, token)
+        yield emit("✅ Done. Query with backlinks.py --repo " + repo)
+    except Exception as e:
+        yield emit(f"❌ FAILED: {e}")
+    finally:
+        builtins.print = orig_print
+
+
+with gr.Blocks(title="CC domain backlink graph builder") as demo:
+    gr.Markdown(
+        "# Common Crawl domain backlink-graph builder\n"
+        f"Transforms the **{RELEASE}** domain hyperlink graph into partitioned "
+        "Parquet on a Hugging Face dataset repo, queryable remotely with DuckDB.\n\n"
+        "Set `HF_TOKEN` and `REPO` as Space secrets, then build."
+    )
+    repo_in = gr.Textbox(label="Target dataset repo", placeholder="you/cc-domain-graph",
+                         value=os.environ.get("REPO", ""))
+    btn = gr.Button("Build & push", variant="primary")
+    out = gr.Textbox(label="Progress", lines=20, max_lines=40)
+    btn.click(run, inputs=repo_in, outputs=out)
+
+if __name__ == "__main__":
+    demo.queue().launch()

build.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""One-time build: transform a Common Crawl domain hyperlink graph into a
+partitioned Parquet dataset on Hugging Face, queryable remotely for backlinks.
+
+Streams the graph straight from CC's CDN (no 24 GB local copy of the source —
+DuckDB reads the gzipped CSVs over httpfs in a single pass). Output (~25-30 GB
+of Parquet) is written locally, then pushed to an HF dataset repo.
+
+Run anywhere with ~30 GB free disk + an HF token:
+    HF_TOKEN=hf_xxx python3 build.py --repo isam/cc-domain-graph
+
+Produces on the repo:
+    manifest.json
+    vertices_by_domain.parquet   (id, rev_domain)   sorted by rev_domain
+    vertices_by_id.parquet       (id, rev_domain)   sorted by id
+    ranks_by_id.parquet          (id, harmonic_pos)
+    edges/part=<n>/data_0.parquet (from_id, to_id)  partitioned by to_id // PART_SIZE
+"""
+
+import argparse
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+
+import duckdb
+
+RELEASE = "cc-main-2025-oct-nov-dec"
+BASE = f"https://data.commoncrawl.org/projects/hyperlinkgraph/{RELEASE}/domain"
+V_URL = f"{BASE}/{RELEASE}-domain-vertices.txt.gz"
+E_URL = f"{BASE}/{RELEASE}-domain-edges.txt.gz"
+R_URL = f"{BASE}/{RELEASE}-domain-ranks.txt.gz"
+
+# ~121M domain nodes → ~1M ids per partition → ~122 partition files.
+PART_SIZE = 1_000_000
+
+
+def log(msg: str):
+    print(msg, flush=True)
+
+
+def build(out_dir: Path, memory_limit: str = "12GB") -> dict:
+    out_dir.mkdir(parents=True, exist_ok=True)
+    (out_dir / "edges").mkdir(exist_ok=True)
+    tmp = out_dir / "_duck_tmp"
+    tmp.mkdir(exist_ok=True)
+
+    con = duckdb.connect(config={"memory_limit": memory_limit, "temp_directory": str(tmp)})
+    con.execute("INSTALL httpfs; LOAD httpfs;")
+
+    # ---- vertices: id, rev_domain (3rd col n_hosts ignored) ----
+    log("Loading vertices (851 MiB gz) ...")
+    con.execute(f"""
+        CREATE TABLE v AS
+        SELECT c0::UINTEGER AS id, c1 AS rev_domain
+        FROM read_csv('{V_URL}', delim='\t', header=false, auto_detect=false,
+                      columns={{'c0':'UINTEGER','c1':'VARCHAR','c2':'UBIGINT'}})
+    """)
+    n_nodes = con.execute("SELECT count(*) FROM v").fetchone()[0]
+    log(f"  {n_nodes:,} domain nodes")
+    con.execute(f"COPY (SELECT * FROM v ORDER BY rev_domain) TO '{out_dir}/vertices_by_domain.parquet' (FORMAT parquet)")
+    con.execute(f"COPY (SELECT * FROM v ORDER BY id) TO '{out_dir}/vertices_by_id.parquet' (FORMAT parquet)")
+    log("  wrote vertices_by_domain.parquet, vertices_by_id.parquet")
+
+    # ---- ranks: harmonicc_pos keyed by host_rev → join to id ----
+    log("Loading ranks ...")
+    con.execute(f"""
+        CREATE TABLE r AS
+        SELECT c0::UBIGINT AS harmonic_pos, c4 AS rev_domain
+        FROM read_csv('{R_URL}', delim='\t', header=false, skip=1, auto_detect=false,
+                      columns={{'c0':'UBIGINT','c1':'VARCHAR','c2':'UBIGINT',
+                                'c3':'VARCHAR','c4':'VARCHAR','c5':'UBIGINT'}})
+    """)
+    con.execute(f"""
+        COPY (SELECT v.id, r.harmonic_pos
+              FROM r JOIN v USING (rev_domain) ORDER BY v.id)
+        TO '{out_dir}/ranks_by_id.parquet' (FORMAT parquet)
+    """)
+    log("  wrote ranks_by_id.parquet")
+
+    # ---- edges: partition by to_id // PART_SIZE (no global sort needed,
+    #      each partition already spans a narrow to_id range) ----
+    log("Streaming + partitioning edges (22.9 GiB gz, single pass) — this is the long step ...")
+    con.execute(f"""
+        COPY (SELECT c0::UINTEGER AS from_id, c1::UINTEGER AS to_id,
+                     (c1::UBIGINT // {PART_SIZE})::UINTEGER AS part
+              FROM read_csv('{E_URL}', delim='\t', header=false, auto_detect=false,
+                            columns={{'c0':'UINTEGER','c1':'UINTEGER'}}))
+        TO '{out_dir}/edges' (FORMAT parquet, PARTITION_BY (part), OVERWRITE_OR_IGNORE)
+    """)
+    n_parts = len(list((out_dir / "edges").glob("part=*")))
+    log(f"  wrote {n_parts} edge partitions")
+
+    shutil.rmtree(tmp, ignore_errors=True)
+
+    manifest = {
+        "release": RELEASE,
+        "source": BASE,
+        "part_size": PART_SIZE,
+        "domain_nodes": n_nodes,
+        "schema": {
+            "vertices": ["id", "rev_domain"],
+            "edges": ["from_id", "to_id"],
+            "ranks": ["id", "harmonic_pos"],
+        },
+        "note": "rev_domain is label-reversed (blackstump.com.au -> au.com.blackstump). "
+                "Backlinks: read edges/part=<to_id//part_size> WHERE to_id=<id>.",
+    }
+    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))
+    log("  wrote manifest.json")
+    return manifest
+
+
+def push(out_dir: Path, repo: str, token: str):
+    from huggingface_hub import HfApi
+    api = HfApi(token=token)
+    api.create_repo(repo, repo_type="dataset", exist_ok=True)
+    log(f"Uploading {out_dir} → {repo} (dataset) ...")
+    # upload_large_folder handles many files + resumes; fall back to upload_folder.
+    try:
+        api.upload_large_folder(folder_path=str(out_dir), repo_id=repo, repo_type="dataset")
+    except AttributeError:
+        api.upload_folder(folder_path=str(out_dir), repo_id=repo, repo_type="dataset")
+    log("Done. Query with:  python3 backlinks.py <domain> --repo " + repo)
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Build the hosted CC domain backlink graph.")
+    ap.add_argument("--repo", required=True, help="Target HF dataset repo, e.g. isam/cc-domain-graph")
+    ap.add_argument("--out", default="./graph_out", help="Local output dir (default: ./graph_out)")
+    ap.add_argument("--memory-limit", default="12GB", help="DuckDB memory limit (default: 12GB)")
+    ap.add_argument("--no-push", action="store_true", help="Build locally only, skip HF upload")
+    args = ap.parse_args()
+
+    out = Path(args.out)
+    build(out, memory_limit=args.memory_limit)
+
+    if args.no_push:
+        log(f"Built in {out} (not pushed).")
+        return
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        sys.exit("Set HF_TOKEN env var to push (or use --no-push).")
+    push(out, args.repo, token)
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,3 @@
+duckdb>=1.0
+huggingface_hub>=0.25
+gradio>=4.0