Upload folder using huggingface_hub

src/search.py

@@ -12,8 +12,9 @@ from rank_bm25 import BM25Okapi
 load_dotenv()
 
 from src.embeddings import Embedder, default_min_score, embeddings_filename
+from src.storage import resolve_data_dir
 
-DATA_DIR = Path(__file__).resolve().parent.parent / "data"
+DATA_DIR = resolve_data_dir()
 
 RRF_K = 60
 HYBRID_TOP_N = 100

src/server.py

@@ -21,26 +21,29 @@ engine: SearchEngine | None = None
 
 
 def _embedding_files_to_fetch() -> list[str]:
-    """Return filenames that should be pulled from HF Hub (only when configured)."""
-    if not os.environ.get("HF_DATASET_REPO"):
-        return []
+    """Filenames to ensure locally (prefer multi-field for the active provider)."""
     provider = get_provider()
-    return [
-        embeddings_filename(provider, multi=True),
-        embeddings_filename(provider, multi=False),
-    ]
+    return [embeddings_filename(provider, multi=True)]
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     global engine
-    files = _embedding_files_to_fetch()
-    if files:
-        from src.storage import ensure_embeddings
-        ensure_embeddings(files)
+    from src.storage import ensure_embeddings, resolve_data_dir
+
+    # On Vercel, /var/task is read-only and azure .npy is gitignored — resolve a
+    # writable data dir and pull embeddings from HF Hub when missing.
+    data_dir = resolve_data_dir()
+    os.environ["DATA_DIR"] = str(data_dir)
+    ensure_embeddings(_embedding_files_to_fetch(), data_dir=data_dir)
+
+    import src.search as search_mod
+    search_mod.DATA_DIR = data_dir
+
     engine = SearchEngine()
     print(f"Loaded {len(engine.companies)} companies with embeddings of shape {engine.embeddings.shape}")
     print(f"Embedding provider: {engine.embedder.provider}")
+    print(f"Data dir: {data_dir}")
     yield
 
 

src/storage.py

@@ -1,9 +1,11 @@
 """Download embedding files from Hugging Face Hub Dataset at startup if not present locally."""
 
 import os
+import shutil
 from pathlib import Path
 
-DATA_DIR = Path(__file__).resolve().parent.parent / "data"
+# Bundled/repo data directory (read-only on Vercel/Lambda).
+REPO_DATA_DIR = Path(__file__).resolve().parent.parent / "data"
 
 
 def _is_valid_npy(path: Path) -> bool:
@@ -15,12 +17,71 @@ def _is_valid_npy(path: Path) -> bool:
         return False
 
 
-def ensure_embeddings(filenames: list[str]) -> None:
-    """Download any missing or corrupted embedding files from HF Hub Dataset."""
-    missing = [f for f in filenames if not (DATA_DIR / f).exists() or not _is_valid_npy(DATA_DIR / f)]
-    if not missing:
-        print("Embeddings already present locally, skipping download.")
+def _is_writable(path: Path) -> bool:
+    try:
+        path.mkdir(parents=True, exist_ok=True)
+        probe = path / ".write_test"
+        probe.write_text("ok")
+        probe.unlink()
+        return True
+    except Exception:
+        return False
+
+
+def resolve_data_dir() -> Path:
+    """Return a writable data dir. On Vercel, /var/task is read-only so use /tmp."""
+    if explicit := os.environ.get("DATA_DIR"):
+        p = Path(explicit)
+        p.mkdir(parents=True, exist_ok=True)
+        return p
+    if _is_writable(REPO_DATA_DIR):
+        return REPO_DATA_DIR
+    tmp = Path("/tmp/nichefind-data")
+    tmp.mkdir(parents=True, exist_ok=True)
+    return tmp
+
+
+def ensure_companies_json(data_dir: Path) -> None:
+    """Make sure companies.json is available in data_dir (copy from bundle if needed)."""
+    dest = data_dir / "companies.json"
+    if dest.exists():
         return
+    src = REPO_DATA_DIR / "companies.json"
+    if not src.exists():
+        raise FileNotFoundError(f"Missing companies.json at {src}")
+    if src.resolve() != dest.resolve():
+        shutil.copy2(src, dest)
+        print(f"Copied companies.json → {dest}")
+
+
+def ensure_embeddings(filenames: list[str], data_dir: Path | None = None) -> Path:
+    """Download any missing or corrupted embedding files from HF Hub Dataset.
+
+    Returns the data directory where files live.
+    """
+    data_dir = data_dir or resolve_data_dir()
+    ensure_companies_json(data_dir)
+
+    missing = [
+        f for f in filenames
+        if not (data_dir / f).exists() or not _is_valid_npy(data_dir / f)
+    ]
+    # Also check the bundled repo dir (CLI deploys that include the .npy locally)
+    if missing:
+        still_missing = []
+        for f in missing:
+            bundled = REPO_DATA_DIR / f
+            if bundled.exists() and _is_valid_npy(bundled):
+                if bundled.resolve() != (data_dir / f).resolve():
+                    shutil.copy2(bundled, data_dir / f)
+                    print(f"Copied {f} → {data_dir / f}")
+            else:
+                still_missing.append(f)
+        missing = still_missing
+
+    if not missing:
+        print("Embeddings already present, skipping download.")
+        return data_dir
 
     from huggingface_hub import hf_hub_download
 
@@ -29,18 +90,20 @@ def ensure_embeddings(filenames: list[str]) -> None:
 
     if not repo_id:
         raise RuntimeError(
-            "Missing HF_DATASET_REPO env var. Set it to your HF dataset repo, e.g. 'yourname/nichefind-data'"
+            "Missing embeddings and HF_DATASET_REPO is not set. "
+            "Either bundle embeddings.azure.multi.npy or set HF_DATASET_REPO."
         )
 
     print(f"Downloading {len(missing)} embedding file(s) from HF dataset '{repo_id}' ...")
     for filename in missing:
-        dest = DATA_DIR / filename
+        dest = data_dir / filename
         print(f"  ↓ {filename}")
         hf_hub_download(
             repo_id=repo_id,
             filename=filename,
             repo_type="dataset",
             token=token,
-            local_dir=str(DATA_DIR),
+            local_dir=str(data_dir),
         )
         print(f"  ✓ {filename} ({dest.stat().st_size / 1e6:.1f} MB)")
+    return data_dir

vercel.json

@@ -4,8 +4,8 @@
   ],
   "functions": {
     "api/index.py": {
-      "maxDuration": 60,
-      "includeFiles": "data/{companies.json,embeddings.azure.multi.npy},static/**"
+      "maxDuration": 300,
+      "includeFiles": "{data/companies.json,static/**}"
     }
   }
 }