Update ingest.py

ingest.py

@@ -1,24 +1,31 @@
 import os
+import shutil
 from git import Repo
 from chunker import chunk_code
 
-SUPPORTED_EXT = (".py", ".js", ".java", ".cpp")
+SUPPORTED_EXTENSIONS = (".py", ".js", ".java", ".cpp")
+BASE_REPO_DIR = "/tmp/user_repo"
 
-def load_repo(repo_url, local_dir="repo"):
-    if os.path.exists(local_dir):
-        return local_dir
-    Repo.clone_from(repo_url, local_dir)
-    return local_dir
+def load_repo(repo_url: str) -> str:
+    # Always start fresh
+    if os.path.exists(BASE_REPO_DIR):
+        shutil.rmtree(BASE_REPO_DIR)
 
-def ingest_repo(repo_path):
+    Repo.clone_from(repo_url, BASE_REPO_DIR)
+    return BASE_REPO_DIR
+
+def ingest_repo(repo_path: str):
     documents = []
 
     for root, _, files in os.walk(repo_path):
         for file in files:
-            if file.endswith(SUPPORTED_EXT):
-                path = os.path.join(root, file)
-                with open(path, "r", errors="ignore") as f:
-                    code = f.read()
-                    documents.extend(chunk_code(path, code))
+            if file.endswith(SUPPORTED_EXTENSIONS):
+                file_path = os.path.join(root, file)
+                try:
+                    with open(file_path, "r", errors="ignore") as f:
+                        code = f.read()
+                        documents.extend(chunk_code(file_path, code))
+                except Exception:
+                    pass
 
     return documents