app.py

"""Hugging Face Space: one-click build of the CC domain backlink graph.

Set two Space secrets:
  HF_TOKEN  — a write token for the target dataset repo
  REPO      — target dataset repo id, e.g. "isam/cc-domain-graph"

Then open the Space and click "Build & push". Progress streams below.
The build streams the source from CC (no large source download to the Space);
it writes ~25-30 GB of Parquet locally, so give the Space enough ephemeral
disk (free tier is usually fine; upgrade hardware if it runs out of space).
"""

import os
from pathlib import Path

import gradio as gr

from build import build, push, RELEASE


def run(repo: str):
    repo = (repo or os.environ.get("REPO") or "").strip()
    token = os.environ.get("HF_TOKEN")
    logs = []

    def emit(line):
        logs.append(line)
        return "\n".join(logs)

    if not repo:
        yield emit("ERROR: set REPO secret or type a repo id (e.g. you/cc-domain-graph)")
        return
    if not token:
        yield emit("ERROR: set the HF_TOKEN Space secret (write access).")
        return

    yield emit(f"Building {RELEASE} → {repo}")
    out = Path("/tmp/graph_out")

    # build() prints with flush; capture by redirecting print via a simple shim.
    import builtins
    orig_print = builtins.print
    buffer = {"last": ""}

    def teed_print(*a, **k):
        msg = " ".join(str(x) for x in a)
        buffer["last"] = msg
        orig_print(*a, **k)

    builtins.print = teed_print
    try:
        manifest = build(out)
        yield emit(f"Built {manifest['domain_nodes']:,} nodes. Uploading ...")
        push(out, repo, token)
        yield emit("✅ Done. Query with backlinks.py --repo " + repo)
    except Exception as e:
        yield emit(f"❌ FAILED: {e}")
    finally:
        builtins.print = orig_print


with gr.Blocks(title="CC domain backlink graph builder") as demo:
    gr.Markdown(
        "# Common Crawl domain backlink-graph builder\n"
        f"Transforms the **{RELEASE}** domain hyperlink graph into partitioned "
        "Parquet on a Hugging Face dataset repo, queryable remotely with DuckDB.\n\n"
        "Set `HF_TOKEN` and `REPO` as Space secrets, then build."
    )
    repo_in = gr.Textbox(label="Target dataset repo", placeholder="you/cc-domain-graph",
                         value=os.environ.get("REPO", ""))
    btn = gr.Button("Build & push", variant="primary")
    out = gr.Textbox(label="Progress", lines=20, max_lines=40)
    btn.click(run, inputs=repo_in, outputs=out)

if __name__ == "__main__":
    demo.queue().launch(server_name="0.0.0.0", server_port=7860)