cloning error fix

src/app.py

@@ -34,43 +34,51 @@ def _init_submodule() -> None:
             raise RuntimeError("GH_TOKEN secret is not set.")
 
         import shutil
+        import time as _time
 
-        # Aggressively remove any partial/corrupt lsp directory
+        # Clean any stale state
         if _LSP_PATH.exists():
             shutil.rmtree(str(_LSP_PATH), ignore_errors=True)
-        # Also nuke any leftover .git/modules/lsp entry
         git_modules = _BASE / ".git" / "modules" / "lsp"
         if git_modules.exists():
             shutil.rmtree(str(git_modules), ignore_errors=True)
 
+        # Clone to /tmp first (always writable, avoids HF Space fs quirks),
+        # then copy to /app/lsp. If shallow clone fails, fall back to full.
         clone_url = f"https://ehejin:{token}@github.com/batu-el/lsp.git"
+        tmp_lsp   = Path("/tmp/lsp_clone")
 
-        for attempt in range(1, 4):
-            print(f"[SUBMODULE] clone attempt {attempt}/3 ...")
-            # Remove any partial clone from previous attempt
-            if _LSP_PATH.exists():
-                shutil.rmtree(str(_LSP_PATH), ignore_errors=True)
-            result = subprocess.run(
-                [
-                    "git", "clone",
-                    "--branch", "0412_train",
-                    "--depth", "1",
-                    clone_url,
-                    str(_LSP_PATH),
-                ],
-                capture_output=True, text=True,
-            )
+        for attempt in range(1, 6):
+            use_shallow = attempt <= 3
+            print(f"[SUBMODULE] clone attempt {attempt}/5 to /tmp "
+                  f"({'shallow' if use_shallow else 'full'})...")
+
+            if tmp_lsp.exists():
+                shutil.rmtree(str(tmp_lsp), ignore_errors=True)
+
+            cmd = ["git", "clone", "--branch", "0412_train"]
+            if use_shallow:
+                cmd += ["--depth", "1"]
+            cmd += [clone_url, str(tmp_lsp)]
+
+            result = subprocess.run(cmd, capture_output=True, text=True)
             print(f"[SUBMODULE] returncode: {result.returncode}")
             if result.stderr:
-                # Scrub token from log
                 print(f"[SUBMODULE] stderr: {result.stderr.replace(token, '***')}")
-            if result.returncode == 0 and (_LSP_PATH / "src" / "prompts").exists():
-                print("[SUBMODULE] clone succeeded.")
+
+            if result.returncode == 0 and (tmp_lsp / "src" / "prompts").exists():
+                # Copy from /tmp to /app/lsp
+                print("[SUBMODULE] clone succeeded, copying to /app/lsp...")
+                shutil.copytree(str(tmp_lsp), str(_LSP_PATH))
+                shutil.rmtree(str(tmp_lsp), ignore_errors=True)
+                print("[SUBMODULE] ready.")
                 break
-            print(f"[SUBMODULE] attempt {attempt} failed, retrying...")
+
+            print(f"[SUBMODULE] attempt {attempt} failed, waiting 3s...")
+            _time.sleep(3)
         else:
             raise RuntimeError(
-                f"Failed to clone lsp after 3 attempts. "
+                f"Failed to clone lsp after 5 attempts. "
                 f"Last stderr: {result.stderr.replace(token, '***')}"
             )
 
@@ -83,6 +91,17 @@ def _init_submodule() -> None:
 
 _init_submodule()
 
+# Wipe any stale local state on container startup.
+# Completions stay durable in HF; we re-scan HF fresh after wipe.
+_data_root = _BASE / "data"
+for pattern in ("reservations.json", "local_completions_*.json", "completion_cache_*.json"):
+    for f in _data_root.glob(pattern):
+        try:
+            f.unlink()
+            print(f"[STARTUP] Wiped stale file: {f.name}")
+        except Exception as e:
+            print(f"[STARTUP] Could not wipe {f.name}: {e}")
+
 # ---------------------------------------------------------------------------
 # 2. App imports (only after submodule is initialised)
 # ---------------------------------------------------------------------------
@@ -108,13 +127,9 @@ from src.ui.screens_preference import screen_pair_intro
 # 3. Admin dashboard — visit ?admin=1
 # ---------------------------------------------------------------------------
 def _screen_admin(cfg: dict) -> None:
-    """
-    Coverage dashboard — visit ?admin=1 to see this.
-    Always scans the HF repo directly — ignores local completions cache
-    so the count reflects real accepted submissions only.
-    """
+    """Coverage dashboard — visit ?admin=1 to see this."""
     from src.data import (
-        _load_pool, _pool_path, _data_dir,
+        _get_accepted_counts, _load_pool, _pool_path,
         _load_reservations, _expire_reservations,
     )
 
@@ -126,74 +141,29 @@ def _screen_admin(cfg: dict) -> None:
     )
 
     if st.button("🔄 Refresh", type="primary"):
+        # Invalidate HF completion caches so we re-scan
+        from src.data import _data_dir
+        for f in _data_dir(cfg).glob("completion_cache*"):
+            f.unlink()
         st.rerun()
 
-    hf_token    = cfg.get("hf_token", "")
-    output_repo = cfg.get("output_dataset_repo", "")
-
     for cat_cfg in cfg["categories"]:
-        cat  = cat_cfg["name"]
-        pool = _load_pool(str(_pool_path(cat, cfg)))
+        cat   = cat_cfg["name"]
+        pool  = _load_pool(str(_pool_path(cat, cfg)))
         total = len(pool)
 
-        # ── Scan HF directly (no cache) ──────────────────────────────────────
-        hf_counts = {str(i): 0 for i in range(total)}
-        n_json    = 0
-        if hf_token and output_repo:
-            try:
-                from huggingface_hub import HfApi
-                api        = HfApi(token=hf_token)
-                files      = list(api.list_repo_files(repo_id=output_repo, repo_type="dataset"))
-                json_files = [f for f in files if f.startswith("json/") and f.endswith(".json")]
-                n_json     = len(json_files)
-                # Build a pair_id → pool_index lookup for fallback matching
-                id_to_index = {}
-                for i, p in enumerate(pool):
-                    pid = p.get("pair_id") or p.get("item_id", "")
-                    if pid:
-                        id_to_index[pid] = i
-
-                for filepath in json_files:
-                    try:
-                        content = api.hf_hub_download(
-                            repo_id=output_repo,
-                            filename=filepath,
-                            repo_type="dataset",
-                            token=hf_token,
-                        )
-                        with open(content) as f:
-                            submission = json.load(f)
-                        for item in submission.get("items", []):
-                            if item.get("category") != cat:
-                                continue
-                            # Use _pool_index if present (new submissions),
-                            # fall back to pair_id/item_id matching (old submissions)
-                            idx = item.get("_pool_index")
-                            if idx is None:
-                                pid = item.get("pair_id") or item.get("item_id", "")
-                                idx = id_to_index.get(pid)
-                            if idx is not None:
-                                hf_counts[str(idx)] = hf_counts.get(str(idx), 0) + 1
-                    except Exception as e:
-                        st.warning(f"Could not parse {filepath}: {e}")
-            except Exception as e:
-                st.error(f"Could not scan HF repo: {e}")
-
-        # ── Reservations (active in-progress users) ───────────────────────────
+        counts       = _get_accepted_counts(cat, cfg)
         reservations = _load_reservations(cfg)
         _expire_reservations(reservations)
+
+        covered = sum(1 for v in counts.values() if v >= 1)
         reserved_uncovered = sum(
-            1 for k, v in reservations.items()
-            if hf_counts.get(k, 0) == 0
+            1 for k in reservations
+            if counts.get(k, 0) == 0
         )
-
-        covered         = sum(1 for v in hf_counts.values() if v >= 1)
-        uncovered       = total - covered
-        truly_uncovered = uncovered - reserved_uncovered
+        truly_uncovered = total - covered - reserved_uncovered
 
         st.markdown(f"### {cat.capitalize()}")
-        st.caption(f"{n_json} submission file(s) in HF repo")
-
         col1, col2, col3, col4 = st.columns(4)
         col1.metric("Total items",     total)
         col2.metric("Covered ✅",      covered)
@@ -206,15 +176,11 @@ def _screen_admin(cfg: dict) -> None:
         if truly_uncovered == 0 and reserved_uncovered == 0:
             st.success(f"✅ All {total} items covered!")
         elif truly_uncovered == 0:
-            st.info(
-                f"🔄 {reserved_uncovered} item(s) in progress — "
-                f"waiting for active participants to finish."
-            )
+            st.info(f"🔄 {reserved_uncovered} item(s) in progress.")
         else:
             st.warning(
                 f"⚠️ {truly_uncovered} item(s) still need a participant. "
-                f"Send more Prolific slots or wait for in-progress "
-                f"reservations to expire (up to 80 min)."
+                f"Send more Prolific slots."
             )
 
         st.markdown("---")