p4_stop_probe: dated scratch ckpt repo per attempt instead of clearing one (v3 died on HfApi.delete_folder's argument order in 20 s); define PROBE before the argv that uses it

kernels/p4_stop_probe.py

@@ -48,7 +48,11 @@ import hubckpt
 from huggingface_hub import HfApi
 
 MIX = "Cion-lab/ounce100m-mix-v1"
-PROBE = "Cion-lab/ounce100m-ckpt-probe"          # never the real run's repo, see the header
+# A scratch repo per attempt (dated), never the real run's repo: v1 left `final` at step 20 and v2 was
+# correctly refused by the stale-stop guard, so rather than clearing state between attempts each one gets an
+# empty repo. That also makes every attempt walk the RepoMissing resume branch (E-034) that session 1 hits.
+PROBE = os.environ.get("P4_PROBE_REPO") or (
+    "Cion-lab/ounce100m-ckpt-probe-" + time.strftime("%m%d-%H%M", time.gmtime()))
 ROOT, RUN = "/kaggle/working/mixroot", "/kaggle/working/run"
 # Two modes, one file, so the assertions are literally the same code in both. `smoke` is the user's
 # suggestion and it is the right order: 20 steps costs ~12 minutes and answers "does the training code run
@@ -126,7 +130,8 @@ rc, out = run([sys.executable, "-c",
 if rc != 0:
     raise SystemExit("VERDICT P4PROBE_STOP could not fetch the published mix")
 man = json.load(open(os.path.join(ROOT, "manifest.json")))
-print("mix", man["n_shards"], "shards", format(int(man["total_tokens"]), ","), "tokens", flush=True)
+print("mix", man["n_shards"], "shards", format(int(man["total_tokens"]), ","), "tokens",
+      "| probe repo", PROBE, flush=True)
 
 common = ["train_ounce100m.py", "--root", ROOT, "--out", RUN,
           "--hub-repo", PROBE, "--prune", "--seq-len", "1024", "--attn", "eager",
@@ -149,43 +154,7 @@ def diag(tag):
     return out
 
 
-def reset_probe_repo():
-    """Empty the scratch checkpoint repo before leg 1.
-
-    Probe 1 v2 was refused at 48 s for exactly the right reason: v1 had left `final` at step 20 in this
-    repo, so `--stop-after-steps 10` is not beyond the resume point and the new guard stopped it. The
-    refusal is the guard working; the dirty repo is what this reset exists for.
-    """
-    api = HfApi(token=os.environ["HF_TOKEN"])
-    try:
-        names = list(api.list_repo_files(PROBE, repo_type="dataset"))
-    except Exception as e:
-        print("probe repo unreadable (%s) -- creating it fresh" % type(e).__name__, flush=True)
-        api.create_repo(PROBE, repo_type="dataset", private=False, exist_ok=True)
-        names = []
-    gone = []
-    for folder in ("ckpt", "final"):
-        if any(n.startswith(folder + "/") for n in names):
-            try:
-                api.delete_folder(PROBE, path_in_repo=folder, repo_type="dataset")
-                gone.append(folder + "/")
-            except Exception as e:
-                print("  delete_folder %s: %s %s" % (folder, type(e).__name__, str(e)[:120]), flush=True)
-    if "latest.json" in names:
-        try:
-            api.delete_file(PROBE, path_in_repo="latest.json", repo_type="dataset")
-            gone.append("latest.json")
-        except Exception as e:
-            print("  delete_file latest.json: %s %s" % (type(e).__name__, str(e)[:120]), flush=True)
-    ptr = hubckpt.latest_pointer(PROBE, token=os.environ["HF_TOKEN"])
-    print("PROBE_RESET deleted", gone, "pointer now",
-          {k: ptr.get(k) for k in ("step", "not_found", "error")}, flush=True)
-    if not ptr.get("not_found") and int(ptr.get("step") or 0) != 0:
-        raise SystemExit("the scratch repo still points at step %s after the reset -- refusing to run a "
-                         "probe whose first leg would resume from a previous attempt" % ptr.get("step"))
-
-
-reset_probe_repo()
+print("PROBE_REPO", PROBE, flush=True)
 shutil.rmtree(RUN, ignore_errors=True)
 rc1, o1 = run(TORCHRUN + common + ["--stop-after-steps", str(STOP1)], "LEG1_STOP_EARLY", T_LEG1)
 if rc1 != 0: