streamline app

requirements.txt

@@ -1,6 +1,7 @@
 duckdb
 pandas
 streamlit
+python-dotenv
 sentence-transformers
 langchain
 langchain-community

src/streamlit_app.py

@@ -16,6 +16,9 @@ from utils.retrieval_helpers import enrich_search_results, enrich_bm25_search_re
 from utils.bm25 import load
 from utils.semantic import load_vector_store
 
+from dotenv import load_dotenv
+load_dotenv()
+
 import warnings
 warnings.filterwarnings("ignore", category=UserWarning)
 
@@ -31,50 +34,10 @@ st.set_page_config(
 FEEDBACK_CSV = ROOT / "results" / "feedback.csv"
 FEEDBACK_CSV.parent.mkdir(parents=True, exist_ok=True)
 
-# ─── Load HF dataset (cached so it only runs once) ───────────────────────────
-from datasets import load_dataset
-
-@st.cache_resource
-def load_hf_dataset():
-    return load_dataset(
-        "McAuley-Lab/Amazon-Reviews-2023",
-        "raw_meta_Grocery_and_Gourmet_Food",
-        trust_remote_code=True,
-        token=os.environ.get("HF_TOKEN")
-    )
-
-HF_DATASET = load_hf_dataset()
-
-# ─── Download vector store from your HF dataset repo ─────────────────────────
-from huggingface_hub import hf_hub_download, snapshot_download, login
-
-VECTOR_STORE_DIR = ROOT / "embeddings" / "semantic_vector_store"
+HF_TOKEN = os.getenv('HF_TOKEN')
 
-@st.cache_resource
-def load_vector_store_cached():
-    hf_token = os.environ.get("HF_TOKEN")
-    if not hf_token:
-        st.error("HF_TOKEN secret is not set. Go to Space Settings → Secrets.")
-        st.stop()
-
-    login(token=hf_token, add_to_git_credential=False)
-
-    VECTOR_STORE_DIR.mkdir(parents=True, exist_ok=True)
-
-    snapshot_path = snapshot_download(
-        repo_id="rishadaz/amazon_retriever-storage",
-        repo_type="dataset",
-        local_dir=str(VECTOR_STORE_DIR),
-        token=hf_token,
-    )
-
-    mini_index_path    = Path(snapshot_path) / "tokenisation" / "bm25_index_mini.pkl"
-    embeddings_dir     = Path(snapshot_path) / "embeddings"
-
-    vector_store = load_vector_store(embeddings_dir)
-    bm25_retriever = load(mini_index_path)
-
-    return vector_store, bm25_retriever
+from datasets import load_dataset
+from huggingface_hub import snapshot_download, login
 
 # ─── Custom CSS ───────────────────────────────────────────────────────────────
 st.markdown(
@@ -146,15 +109,60 @@ st.markdown(
     unsafe_allow_html=True,
 )
 
-# ─── Placeholder retrieval functions ──────────────────────────────────────────
-# TODO: Replace with real imports once src/bm25.py and src/semantic.py are ready:
-#   from src.bm25 import BM25Retriever
-#   from src.semantic import SemanticRetriever
-#
-# Expected return format — list of dicts with keys:
-#   asin (str), title (str), text (str), rating (float), score (float)
+@st.cache_resource
+def load_hf_dataset():
+    return load_dataset(
+        "McAuley-Lab/Amazon-Reviews-2023",
+        "raw_meta_Grocery_and_Gourmet_Food",
+        trust_remote_code=True,
+        token=HF_TOKEN
+    )
+
+VECTOR_STORE_DIR = ROOT / "data" / "processed"
+
+@st.cache_resource
+def load_vector_store_cached():
+    login(token=HF_TOKEN, add_to_git_credential=False)
+    VECTOR_STORE_DIR.mkdir(parents=True, exist_ok=True)
+
+    snapshot_path = snapshot_download(
+        repo_id="rishadaz/amazon_retriever-storage",
+        repo_type="dataset",
+        local_dir=str(VECTOR_STORE_DIR),
+        token=HF_TOKEN,
+    )
+
+    mini_index_path = Path(snapshot_path) / "tokenisation" / "bm25_index.pkl"
+    embeddings_dir  = Path(snapshot_path) / "embeddings"
+
+    vector_store    = load_vector_store(embeddings_dir)
+    bm25_retriever  = load(mini_index_path)
+
+    return vector_store, bm25_retriever
+
+# ─── Get Data ──────────────────────────────────────────────────────────────
+# local tag will read from your local directory as a default it will
+# read the mini versions of the files we have provided in the repo
+
+data_source = "remote" #"remote" or "local" 
+
+# note: remote has the full generated corpus and 
+# embeddings which can take a long time to download and 
+# the app might become heavy too and slow down 
+# processing. For development pls use the smaller "local" corpus
+
+HF_DATASET = load_hf_dataset()
+
+if data_source == 'local':
+    MINI_INDEX_PATH  = ROOT / "data" / "processed" / "tokenisation" / "bm25_index_mini.pkl"
+
+    vector_store = load_vector_store(ROOT_FOLDER / 'data' / 'processed' / 'embeddings')
+    retriever = load(MINI_INDEX_PATH)
+else:
+    
+    vector_store, retriever = load_vector_store_cached()
+
 
-vector_store, bm25_retriever = load_vector_store_cached()
 
 def bm25_search(query: str, top_k: int = 3) -> list[dict]:
     """
@@ -163,7 +171,8 @@ def bm25_search(query: str, top_k: int = 3) -> list[dict]:
         return retriever.search(query, top_k=top_k)
     Returns top_k review-level results (may include multiple reviews per ASIN).
     """
-    results = enrich_bm25_search_results(bm25_retriever, query, top_k, HF_DATASET['full'])
+
+    results = enrich_bm25_search_results(retriever, query, top_k, HF_DATASET['full'])
     return results
 
 
@@ -216,7 +225,7 @@ def render_results(results: list[dict], mode: str, query: str) -> None:
         title       = item["title"]
         avg_rating  = item["average_rating"]
         n_reviews   = len(reviews)
-        total_reviews = item.get('total_reviews', n_reviews)
+        # total_reviews = item.get('total_reviews', n_reviews)
         rating_number = item.get('rating_number', 0)
         asin        = item['parent_asin']
         review_word = "review" if n_reviews == 1 else "reviews"
@@ -247,7 +256,7 @@ def render_results(results: list[dict], mode: str, query: str) -> None:
         )
 
         # ── Reviews in collapsible expander ───────────────────────────────
-        expander_label = f"📖 View {n_reviews} of total {total_reviews} {review_word} "
+        expander_label = f"📖 Viewing top {n_reviews} {review_word} "
         with st.expander(expander_label, expanded=(n_reviews == 1)):
             for j, rev in enumerate(reviews):
                 st.markdown(

utils/retrieval_helpers.py

@@ -27,28 +27,6 @@ def decode_ratings(page_content):
         return(parsed)
     else:
         return {}
-    
-def decode_bm25_ratings(page_content):
-    block_pattern = r'Review \(Rating:\s*\d+\.\d+\):.*'
-    matches = re.findall(block_pattern, page_content)
-
-    if matches:
-        pattern = r'Review \(Rating:\s*(\d+\.\d+)\):\s*([^\.]+)\.\s*(.*)'
-        parsed = []
-
-        for r in matches[:3]:
-            match = re.match(pattern, r)
-            if match:
-                rating, title, text = match.groups()
-                parsed.append({
-                    'rating': float(rating),
-                    'title': title.strip(),
-                    'text': text.strip()
-                })
-
-        return parsed
-    else:
-        return {}
 
 def enrich_search_results(vector_store, query: str, k: int, hf_dataset):
     """
@@ -146,14 +124,14 @@ def enrich_bm25_search_results(retriever, query: str, k: int, hf_dataset):
 
     for doc, score in results:
         parent_asin = doc.metadata.get("parent_asin")
-        total_reviews = doc.metadata.get("total_reviews")
-        metadata_object = asin_to_metadata.get(parent_asin, {}).copy()
-        metadata_object['score'] = score
-        metadata_object['total_reviews'] = total_reviews
 
-        # 3. Extract reviews from page_content
-        page_content = doc.page_content
-        metadata_object["reviews"] = decode_ratings(page_content)
+        metadata_object = {
+            **doc.metadata,
+            **asin_to_metadata.get(parent_asin, {}),
+            "score": score,
+        }
+
+        metadata_object['reviews'] = metadata_object.pop('top_reviews', {}) or {}
 
         enriched_results.append(metadata_object)