"""Hugging Face Space: one-click build of the CC domain backlink graph. Set two Space secrets: HF_TOKEN — a write token for the target dataset repo REPO — target dataset repo id, e.g. "isam/cc-domain-graph" Then open the Space and click "Build & push". Progress streams below. The build streams the source from CC (no large source download to the Space); it writes ~25-30 GB of Parquet locally, so give the Space enough ephemeral disk (free tier is usually fine; upgrade hardware if it runs out of space). """ import os from pathlib import Path import gradio as gr from build import build, push, RELEASE def run(repo: str): repo = (repo or os.environ.get("REPO") or "").strip() token = os.environ.get("HF_TOKEN") logs = [] def emit(line): logs.append(line) return "\n".join(logs) if not repo: yield emit("ERROR: set REPO secret or type a repo id (e.g. you/cc-domain-graph)") return if not token: yield emit("ERROR: set the HF_TOKEN Space secret (write access).") return yield emit(f"Building {RELEASE} → {repo}") out = Path("/tmp/graph_out") # build() prints with flush; capture by redirecting print via a simple shim. import builtins orig_print = builtins.print buffer = {"last": ""} def teed_print(*a, **k): msg = " ".join(str(x) for x in a) buffer["last"] = msg orig_print(*a, **k) builtins.print = teed_print try: manifest = build(out) yield emit(f"Built {manifest['domain_nodes']:,} nodes. Uploading ...") push(out, repo, token) yield emit("✅ Done. Query with backlinks.py --repo " + repo) except Exception as e: yield emit(f"❌ FAILED: {e}") finally: builtins.print = orig_print with gr.Blocks(title="CC domain backlink graph builder") as demo: gr.Markdown( "# Common Crawl domain backlink-graph builder\n" f"Transforms the **{RELEASE}** domain hyperlink graph into partitioned " "Parquet on a Hugging Face dataset repo, queryable remotely with DuckDB.\n\n" "Set `HF_TOKEN` and `REPO` as Space secrets, then build." ) repo_in = gr.Textbox(label="Target dataset repo", placeholder="you/cc-domain-graph", value=os.environ.get("REPO", "")) btn = gr.Button("Build & push", variant="primary") out = gr.Textbox(label="Progress", lines=20, max_lines=40) btn.click(run, inputs=repo_in, outputs=out) if __name__ == "__main__": demo.queue().launch(server_name="0.0.0.0", server_port=7860)