Update app.py

app.py

@@ -6,6 +6,15 @@ from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 from urllib.parse import urlparse
 import re
+import os, shutil
+
+# Prevent Hugging Face cache from exceeding 50 GB
+os.environ["TRANSFORMERS_CACHE"] = "/tmp"
+os.environ["SENTENCE_TRANSFORMERS_HOME"] = "/tmp"
+
+# Optional: clean old cache if Space restarts
+for folder in ["/root/.cache", "/tmp"]:
+    shutil.rmtree(folder, ignore_errors=True)
 
 st.set_page_config(page_title="404 Redirect Mapper", layout="wide")
 st.title("🔁 404 Redirect Mapper")
@@ -72,9 +81,10 @@ if file_404 and file_200:
             urls_200, text_200 = prepare(df_200, url_col_200, text_cols_200)
 
             status_msg.info("🔄 Generating embeddings...")
-            model = SentenceTransformer("paraphrase-MiniLM-L6-v2")
-            emb_404 = model.encode(text_404, show_progress_bar=True)
-            emb_200 = model.encode(text_200, show_progress_bar=True)
+            model = SentenceTransformer("all-MiniLM-L6-v2")
+            emb_404 = model.encode(text_404, show_progress_bar=True, batch_size=32)
+            emb_200 = model.encode(text_200, show_progress_bar=True, batch_size=32)
+
             status_msg.empty()
 
             sim_matrix = cosine_similarity(emb_404, emb_200)