p1 secret bootstrap: privacy-gated, placeholder-first ordering

probes/p1_secret_bootstrap.py

@@ -0,0 +1,145 @@
+# Phase 1 bootstrap worker: create a PRIVATE Kaggle dataset to hold the HF token, safely.
+#
+# Why this exists (closes D-002). §2 permits the token to come "from the environment or a secret
+# store" and forbids it reaching a public artifact. p1_credential_path proved the Kaggle CLI is
+# preinstalled and already authenticated inside an internet-enabled session, so a job can create its
+# own private dataset and later jobs can mount it from /kaggle/input instead of carrying the token in
+# their source.
+#
+# THE ORDERING HERE IS A SECURITY CONTROL, NOT STYLE. `kaggle datasets create` decides visibility
+# from a metadata field whose name has varied between CLI versions (isPrivate vs defineIsPrivate), and
+# an unrecognised key is silently ignored -- which would create a PUBLIC dataset. A public dataset
+# containing the token would be an unrecoverable leak, so:
+#
+#   1. create with an inert placeholder file only,
+#   2. PROVE it is private by failing to download that file anonymously,
+#   3. only then push the real credential as a new version,
+#   4. re-prove, and delete nothing that was not created here.
+#
+# If step 2 ever shows anonymous read succeeding, the secret is never uploaded and the job says so.
+# No token value is ever printed; kernel stdout is retained. Zero GPU quota (CPU session).
+
+import hashlib
+import json
+import os
+import subprocess
+import time
+import urllib.error
+import urllib.request
+
+R = {}
+DS_SLUG = "ounce100m-secret-store"
+DS_ID = f"dodosoomro/{DS_SLUG}"
+TOKEN_ENV = "OUNCE100M_HF_TOKEN"
+DS_DIR = "/kaggle/working/ds"
+PLACEHOLDER = "placeholder.txt"
+SECRET_FILE = "credentials.json"
+
+
+def scrub(s):
+    import re
+
+    s = s if isinstance(s, str) else str(s)
+    s = re.sub(r"hf_[A-Za-z0-9]{16,}", "<REDACTED_TOKEN>", s)
+    return re.sub(r"[A-Za-z0-9_\-]{36,}", "<REDACTED_LONG>", s)[:1200]
+
+
+def run(cmd, timeout=300):
+    p = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
+    return {"rc": p.returncode, "out": scrub(p.stdout), "err": scrub(p.stderr)}
+
+
+def anon_can_read(path):
+    """True only if an unauthenticated request can fetch the file. That means PUBLIC -- bad."""
+    url = f"https://www.kaggle.com/api/v1/datasets/download/{DS_ID}"
+    try:
+        with urllib.request.urlopen(url, timeout=60) as r:
+            return {"readable": True, "status": r.status, "bytes": len(r.read(2048))}
+    except urllib.error.HTTPError as e:
+        return {"readable": False, "status": e.code, "reason": str(e.reason)[:80]}
+    except Exception as e:
+        return {"readable": None, "error": f"{type(e).__name__}: {e}"[:160]}
+
+
+def write_metadata(private=True):
+    """Both spellings of the visibility key, because the CLI version in the image is 2.0.2 and the
+    field name has moved across releases; an unrecognised key is ignored without error."""
+    md = {
+        "title": "ounce100m runtime assets",
+        "id": DS_ID,
+        "resources": [{"path": PLACEHOLDER, "description": "bootstrap placeholder",
+                       "resourceName": "placeholder", "resourceType": "dataset"}],
+    }
+    if private:
+        md["isPrivate"] = True
+        md["defineIsPrivate"] = True
+    with open(os.path.join(DS_DIR, "dataset-metadata.json"), "w") as f:
+        json.dump(md, f)
+
+
+def write_files(with_secret):
+    os.makedirs(DS_DIR, exist_ok=True)
+    with open(os.path.join(DS_DIR, PLACEHOLDER), "w") as f:
+        f.write("ounce100m bootstrap placeholder\n")
+    if with_secret:
+        tok = os.environ[TOKEN_ENV]
+        with open(os.path.join(DS_DIR, SECRET_FILE), "w") as f:
+            json.dump({"HF_TOKEN": tok}, f)
+        os.chmod(os.path.join(DS_DIR, SECRET_FILE), 0o600)
+
+
+tok = os.environ.get(TOKEN_ENV, "")
+R["pre"] = {"token_present": bool(tok), "token_length": len(tok),
+            "token_sha256_prefix": hashlib.sha256(tok.encode()).hexdigest()[:12] if tok else None}
+
+if not tok:
+    R["fatal"] = f"{TOKEN_ENV} unset; nothing created"
+else:
+    write_files(with_secret=False)
+    write_metadata(private=True)
+
+    existing = run(f"kaggle datasets status {DS_ID} 2>&1", 120)
+    R["pre_existing_status"] = existing
+    already = existing["rc"] == 0
+
+    if not already:
+        t = time.monotonic()
+        R["create"] = run(f"kaggle datasets create -p {DS_DIR} -q 2>&1", timeout=600)
+        R["create"]["seconds"] = round(time.monotonic() - t, 1)
+        time.sleep(12)  # visibility is applied asynchronously; an immediate probe can lie
+
+    # ---- the gate: prove privacy BEFORE any secret exists on the dataset
+    privacy = anon_can_read(PLACEHOLDER)
+    R["privacy_check_placeholder_only"] = privacy
+    R["private_confirmed"] = privacy.get("readable") is False
+
+    if R["private_confirmed"]:
+        write_files(with_secret=True)
+        t = time.monotonic()
+        R["version_with_secret"] = run(
+            f'kaggle datasets version -p {DS_DIR} -m "provision runtime credential" -q 2>&1',
+            timeout=600)
+        R["version_with_secret"]["seconds"] = round(time.monotonic() - t, 1)
+        time.sleep(12)
+        after = anon_can_read(SECRET_FILE)
+        R["privacy_check_after_secret"] = after
+        R["secret_still_private"] = after.get("readable") is False
+        if not R["secret_still_private"]:
+            R["EMERGENCY"] = ("Anonymous read succeeded AFTER the credential was uploaded -- the "
+                              "dataset may be public. Delete dodosoomro/ounce100m-secret-store from "
+                              "the Kaggle web UI immediately and rotate the HF token.")
+    else:
+        R["refused_to_upload"] = ("Dataset was anonymously readable while holding only a placeholder; "
+                                  "the secret was NOT uploaded. Fix visibility handling before retry.")
+
+    R["files"] = run(f"kaggle datasets files {DS_ID} 2>&1", 120)
+    if R.get("secret_still_private"):
+        R["mount_hint_for_future_jobs"] = {
+            "datasetDataSources": [DS_ID],
+            "read_from": f"/kaggle/input/{DS_SLUG}/{SECRET_FILE}",
+            "then": "json.load(...)['HF_TOKEN'] -> os.environ['HF_TOKEN']; never print the value",
+        }
+
+print("PROBE_JSON_BEGIN")
+print(json.dumps(R, indent=1, default=str))
+print("PROBE_JSON_END")