Updated: paths.py to load data from HF Hub

src/utils/paths.py

@@ -1,5 +1,12 @@
 from pathlib import Path
 from typing import Union, Dict
+import os
+
+from huggingface_hub import hf_hub_download
+
+# --- Constants ---
+HF_REPO = "mayankc/mayankc-amazon_beauty_subset"
+CACHE: Dict[str, Path] = {}
 
 # --- project roots ---
 PROJECT_ROOT = Path(__file__).resolve().parents[2]
@@ -10,7 +17,6 @@ CACHE_DIR     = DATA_DIR / "cache"
 LOGS_DIR      = PROJECT_ROOT / "logs"
 MODELS_DIR    = PROJECT_ROOT / "src" / "models"
 
-
 def ensure_dir(path: Union[str, Path]) -> Path:
     """
     Ensure a directory exists. Accepts either a str or a pathlib.Path.
@@ -20,30 +26,50 @@ def ensure_dir(path: Union[str, Path]) -> Path:
     p.mkdir(parents=True, exist_ok=True)
     return p
 
-
 def get_raw_path(dataset: str) -> Path:
     """.../data/raw/<dataset>"""
     return ensure_dir(RAW_DIR / dataset)
 
+def _hf_download(filename: str) -> Path:
+    if filename in CACHE:
+        return CACHE[filename]
+    path = hf_hub_download(repo_id=HF_REPO, filename=filename)
+    CACHE[filename] = Path(path)
+    return Path(path)
 
 def get_processed_path(dataset: str) -> Path:
-    """.../data/processed/<dataset>"""
-    return ensure_dir(PROCESSED_DIR / dataset)
+    """
+    For Hugging Face deployment:
+    - If running locally, use local /data/processed/<dataset>
+    - If on Spaces or missing local files, fall back to hf_hub_download
+    Returns the base processed folder (parent of downloaded file).
+    """
+    local_path = PROCESSED_DIR / dataset
+    if local_path.exists():
+        return local_path
 
+    # fallback to Hugging Face Dataset Hub
+    fallback_file = f"{dataset}/user_text_emb.parquet"
+    fallback_path = _hf_download(fallback_file)
+    return fallback_path.parent
 
 def get_logs_path() -> Path:
-    """.../logs"""
     return ensure_dir(LOGS_DIR)
 
-
 def get_dataset_paths(dataset: str) -> Dict[str, Path]:
     """
-    Convenience bundle of dataset-related paths.
-    NOTE: returns Path objects (not strings) for consistency.
+    Returns dictionary of paths for known dataset assets.
+    If local file not found, pulls from Hugging Face Hub.
     """
     dataset = dataset.lower()
     processed_dir = get_processed_path(dataset)
 
+    def resolve_or_download(name: str) -> Path:
+        local = processed_dir / name
+        if local.exists():
+            return local
+        return _hf_download(f"{dataset}/{name}")
+
     return {
         "raw": get_raw_path(dataset),
         "processed": processed_dir,
@@ -51,16 +77,16 @@ def get_dataset_paths(dataset: str) -> Dict[str, Path]:
         "logs": get_logs_path(),
 
         # Parquet input files
-        "item_meta_emb_path": processed_dir / "item_meta_emb.parquet",
-        "item_image_emb_path": processed_dir / "item_image_emb.parquet",
-        "item_text_emb_path": processed_dir / "item_text_emb.parquet",
+        "item_meta_emb_path": resolve_or_download("item_meta_emb.parquet"),
+        "item_image_emb_path": resolve_or_download("item_image_emb.parquet"),
+        "item_text_emb_path": resolve_or_download("item_text_emb.parquet"),
 
         # FAISS-related npy features
-        "meta_features_path": processed_dir / "meta_features.npy",
-        "text_features_path": processed_dir / "text_features.npy",
-        "image_features_path": processed_dir / "image_features.npy",
-        "labels_path": processed_dir / "labels.json",
+        "meta_features_path": resolve_or_download("meta_features.npy"),
+        "text_features_path": resolve_or_download("text_features.npy"),
+        "image_features_path": resolve_or_download("image_features.npy"),
+        "labels_path": resolve_or_download("labels.json"),
 
-        # ✅ Add missing FAISS fusion output path
-        "faiss_fusion_path": processed_dir / "faiss_fusion.index",
+        # ✅ FAISS fusion output path
+        "faiss_fusion_path": resolve_or_download("faiss_fusion.index"),
     }