saving db files to neon

Co-authored-by: Copilot <copilot@github.com>

.env.example

@@ -1,7 +1,11 @@
 GEMINI_API_KEY=your_gemini_api_key
 GEMINI_MODEL=gemini-2.5-flash
 EMBEDDING_MODEL_NAME=sentence-transformers/all-MiniLM-L6-v2
-STUDENT_PERFORMANCE_URL_TEMPLATE=https://git-connect-backend-v2.vercel.app/api/student/{student_id}/performance
+STUDENT_PERFORMANCE_URL_TEMPLATE=https://git-connect-backend-v2.vercel.app/api/student/{student_id}/performance/{semester}
+RAG_INDEX_DB_URL=
+NEON_MAX_RETRIES=5
+NEON_RETRY_BACKOFF_SEC=1.0
+NEON_CONNECT_TIMEOUT_SEC=10
 VECTOR_DATA_DIR=data/vector_index
 RAW_TEXT_DIR=data/raw_text
 PDF_TIMEOUT_SEC=60

README.md

@@ -24,6 +24,19 @@ FastAPI backend with two primary features:
 - Vector Search: FAISS (IndexFlatIP with normalized vectors)
 - LLM generation/summarization: Gemini
 
+RAG index persistence:
+
+- Runtime retrieval uses semester FAISS files plus semester PKL records.
+- Optional cloud persistence to Neon is enabled with RAG_INDEX_DB_URL.
+- On syllabus processing, updated semester FAISS and PKL are uploaded to Neon.
+- On server restart, if local files are missing, the server hydrates them from Neon automatically.
+
+About stored artifacts:
+
+- Needed for RAG runtime: semester_x.faiss and semester_x.pkl.
+- Raw text files under data/raw_text are not required for search-time RAG.
+- Raw text is mainly useful for debugging/auditing ingestion outputs.
+
 ## Local Setup
 
 1. Create and activate a virtual environment.

app/config.py

@@ -20,8 +20,12 @@ class Settings:
     )
     student_performance_url_template: str = os.getenv(
         "STUDENT_PERFORMANCE_URL_TEMPLATE",
-        "https://git-connect-backend-v2.vercel.app/api/student/{student_id}/performance",
+        "https://git-connect-backend-v2.vercel.app/api/student/{student_id}/performance/{semester}",
     )
+    rag_index_db_url: str = os.getenv("RAG_INDEX_DB_URL", "")
+    neon_max_retries: int = int(os.getenv("NEON_MAX_RETRIES", "5"))
+    neon_retry_backoff_sec: float = float(os.getenv("NEON_RETRY_BACKOFF_SEC", "1.0"))
+    neon_connect_timeout_sec: int = int(os.getenv("NEON_CONNECT_TIMEOUT_SEC", "10"))
     vector_data_dir: str = os.getenv("VECTOR_DATA_DIR", "data/vector_index")
     raw_text_dir: str = os.getenv("RAW_TEXT_DIR", "data/raw_text")
     gemini_model: str = os.getenv("GEMINI_MODEL", "gemini-2.5-flash")

app/main.py

@@ -62,8 +62,13 @@ def process_syllabus(courses: List[CourseInput]) -> SyllabusProcessResponse:
     except ValueError as exc:
         raise HTTPException(status_code=500, detail=str(exc)) from exc
 
-    vector_store = LocalVectorStore(settings.vector_data_dir)
-    os.makedirs(settings.raw_text_dir, exist_ok=True)
+    vector_store = LocalVectorStore(
+        settings.vector_data_dir,
+        rag_index_db_url=settings.rag_index_db_url,
+        neon_max_retries=settings.neon_max_retries,
+        neon_retry_backoff_sec=settings.neon_retry_backoff_sec,
+        neon_connect_timeout_sec=settings.neon_connect_timeout_sec,
+    )
 
     results: List[CourseSummary] = []
     failed: List[CourseProcessError] = []
@@ -79,10 +84,6 @@ def process_syllabus(courses: List[CourseInput]) -> SyllabusProcessResponse:
             if not syllabus_text:
                 raise RuntimeError("No text extracted from PDF.")
 
-            raw_path = os.path.join(settings.raw_text_dir, f"{course.course_code}.txt")
-            with open(raw_path, "w", encoding="utf-8") as f:
-                f.write(syllabus_text)
-
             chunks = chunk_text(syllabus_text)
             if not chunks:
                 raise RuntimeError("Unable to create text chunks from syllabus content.")
@@ -166,7 +167,13 @@ Rules:
 - Be concise and practical.
 """
         elif intent == "syllabus":
-            vector_store = LocalVectorStore(settings.vector_data_dir)
+            vector_store = LocalVectorStore(
+                settings.vector_data_dir,
+                rag_index_db_url=settings.rag_index_db_url,
+                neon_max_retries=settings.neon_max_retries,
+                neon_retry_backoff_sec=settings.neon_retry_backoff_sec,
+                neon_connect_timeout_sec=settings.neon_connect_timeout_sec,
+            )
             query_embedding = gemini.embed_text(req.query, task_type="retrieval_query")
             hits = vector_store.search(req.semester, query_embedding, top_k=5)
             hits = hits[:5]

app/services/neon_index_store.py

@@ -0,0 +1,100 @@
+from typing import Optional, Tuple
+import time
+
+import psycopg
+from psycopg import Binary
+
+
+class NeonIndexStore:
+    def __init__(
+        self,
+        db_url: str,
+        max_retries: int = 5,
+        base_backoff_sec: float = 1.0,
+        connect_timeout_sec: int = 10,
+    ) -> None:
+        self.db_url = db_url
+        self.max_retries = max(1, max_retries)
+        self.base_backoff_sec = max(0.1, base_backoff_sec)
+        self.connect_timeout_sec = max(1, connect_timeout_sec)
+
+    def _connect(self):
+        return psycopg.connect(
+            self.db_url,
+            connect_timeout=self.connect_timeout_sec,
+        )
+
+    def _run_with_retry(self, op_name: str, fn):
+        last_exc: Exception | None = None
+
+        for attempt in range(1, self.max_retries + 1):
+            try:
+                return fn()
+            except Exception as exc:
+                last_exc = exc
+                if attempt < self.max_retries:
+                    sleep_sec = self.base_backoff_sec * (2 ** (attempt - 1))
+                    time.sleep(sleep_sec)
+
+        raise RuntimeError(
+            f"Neon operation '{op_name}' failed after {self.max_retries} attempts: {last_exc}"
+        )
+
+    def ensure_table(self) -> None:
+        query = """
+        CREATE TABLE IF NOT EXISTS rag_semester_indexes (
+            semester INTEGER PRIMARY KEY,
+            faiss_blob BYTEA NOT NULL,
+            records_blob BYTEA NOT NULL,
+            updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+        )
+        """
+
+        def _op():
+            with self._connect() as conn:
+                with conn.cursor() as cur:
+                    cur.execute(query)
+                conn.commit()
+
+        self._run_with_retry("ensure_table", _op)
+
+    def save_semester_files(self, semester: int, faiss_bytes: bytes, records_bytes: bytes) -> None:
+        query = """
+        INSERT INTO rag_semester_indexes (semester, faiss_blob, records_blob, updated_at)
+        VALUES (%s, %s, %s, NOW())
+        ON CONFLICT (semester)
+        DO UPDATE SET
+            faiss_blob = EXCLUDED.faiss_blob,
+            records_blob = EXCLUDED.records_blob,
+            updated_at = NOW()
+        """
+        def _op():
+            with self._connect() as conn:
+                with conn.cursor() as cur:
+                    cur.execute(
+                        query,
+                        (semester, Binary(faiss_bytes), Binary(records_bytes)),
+                    )
+                conn.commit()
+
+        self._run_with_retry("save_semester_files", _op)
+
+    def load_semester_files(self, semester: int) -> Optional[Tuple[bytes, bytes]]:
+        query = """
+        SELECT faiss_blob, records_blob
+        FROM rag_semester_indexes
+        WHERE semester = %s
+        """
+        def _op():
+            with self._connect() as conn:
+                with conn.cursor() as cur:
+                    cur.execute(query, (semester,))
+                    return cur.fetchone()
+
+        row = self._run_with_retry("load_semester_files", _op)
+
+        if not row:
+            return None
+
+        faiss_blob, records_blob = row
+        return bytes(faiss_blob), bytes(records_blob)

app/services/student_service.py

@@ -12,20 +12,41 @@ def fetch_student_info(
         raise ValueError(
             "STUDENT_PERFORMANCE_URL_TEMPLATE must include '{student_id}'."
         )
+    if "{semester}" not in student_performance_url_template:
+        raise ValueError(
+            "STUDENT_PERFORMANCE_URL_TEMPLATE must include '{semester}'."
+        )
+
+    student_url = student_performance_url_template.format(
+        student_id=student_id,
+        semester=semester,
+    )
+    backend_intent = "results" if intent == "result" else intent
 
-    student_url = student_performance_url_template.format(student_id=student_id)
-    get_resp = requests.get(
+    # Contract: GET with intent as URL query parameter.
+    get_with_intent_resp = requests.get(
         student_url,
-        params={"semester": semester, "intent": intent},
+        params={"intent": backend_intent},
         timeout=timeout,
     )
-    if not get_resp.ok:
-        raise RuntimeError(
-            f"Failed to fetch student info from {student_url}; "
-            f"status {get_resp.status_code}."
+    if get_with_intent_resp.ok:
+        return _filter_student_info_by_intent(
+            get_with_intent_resp.json(),
+            semester=semester,
+            intent=intent,
         )
 
-    return _filter_student_info_by_intent(get_resp.json(), semester=semester, intent=intent)
+    # Fallback: plain GET if server ignores intent query param.
+    get_resp = requests.get(student_url, timeout=timeout)
+    if get_resp.ok:
+        return _filter_student_info_by_intent(get_resp.json(), semester=semester, intent=intent)
+
+    raise RuntimeError(
+        "Failed to fetch student info from "
+        f"{student_url}; "
+        f"GET(intent={backend_intent}) status {get_with_intent_resp.status_code}, "
+        f"GET status {get_resp.status_code}."
+    )
 
 
 def _filter_student_info_by_intent(payload: dict, semester: int, intent: str) -> dict:
@@ -36,7 +57,7 @@ def _filter_student_info_by_intent(payload: dict, semester: int, intent: str) ->
         data.pop("results", None)
         return data
 
-    if intent == "result":
+    if intent in {"result", "results"}:
         data.pop("attendance", None)
         results = data.get("results")
         if isinstance(results, dict):

app/vector_store.py

@@ -1,16 +1,45 @@
-import json
 import os
+import pickle
 from typing import List
 
 import faiss
 import numpy as np
 
+from app.services.neon_index_store import NeonIndexStore
+
 
 class LocalVectorStore:
-    def __init__(self, base_dir: str) -> None:
+    def __init__(
+        self,
+        base_dir: str,
+        rag_index_db_url: str = "",
+        neon_max_retries: int = 5,
+        neon_retry_backoff_sec: float = 1.0,
+        neon_connect_timeout_sec: int = 10,
+    ) -> None:
         self.base_dir = base_dir
+        self.cloud_store = (
+            NeonIndexStore(
+                rag_index_db_url,
+                max_retries=neon_max_retries,
+                base_backoff_sec=neon_retry_backoff_sec,
+                connect_timeout_sec=neon_connect_timeout_sec,
+            )
+            if rag_index_db_url
+            else None
+        )
+        self._hydration_error_logged_semesters: set[int] = set()
+        self._setup_error_logged = False
         os.makedirs(self.base_dir, exist_ok=True)
 
+        if self.cloud_store:
+            try:
+                self.cloud_store.ensure_table()
+            except Exception as exc:
+                if not self._setup_error_logged:
+                    print(f"Neon index table setup skipped: {exc}")
+                    self._setup_error_logged = True
+
     def upsert_documents(
         self,
         semester: int,
@@ -18,8 +47,10 @@ class LocalVectorStore:
         chunks: List[str],
         embeddings: List[List[float]],
     ) -> None:
-        meta_path = self._semester_meta_path(semester)
-        records = self._load(meta_path)
+        self._ensure_local_semester_data(semester)
+
+        records_path = self._semester_records_path(semester)
+        records = self._load_records(records_path)
 
         records = [r for r in records if r.get("course_code") != course_code]
         for chunk, vector in zip(chunks, embeddings):
@@ -31,13 +62,16 @@ class LocalVectorStore:
                 }
             )
 
-        self._save(meta_path, records)
+        self._save_records(records_path, records)
         self._rebuild_faiss_index(semester, records)
+        self._sync_semester_to_cloud(semester)
 
     def search(self, semester: int, query_embedding: List[float], top_k: int = 6) -> List[dict]:
-        meta_path = self._semester_meta_path(semester)
+        self._ensure_local_semester_data(semester)
+
+        records_path = self._semester_records_path(semester)
         index_path = self._semester_index_path(semester)
-        records = self._load(meta_path)
+        records = self._load_records(records_path)
         if not records:
             return []
 
@@ -70,8 +104,8 @@ class LocalVectorStore:
 
         return hits
 
-    def _semester_meta_path(self, semester: int) -> str:
-        return os.path.join(self.base_dir, f"semester_{semester}.meta.json")
+    def _semester_records_path(self, semester: int) -> str:
+        return os.path.join(self.base_dir, f"semester_{semester}.pkl")
 
     def _semester_index_path(self, semester: int) -> str:
         return os.path.join(self.base_dir, f"semester_{semester}.faiss")
@@ -94,14 +128,60 @@ class LocalVectorStore:
         index.add(vectors)
         faiss.write_index(index, index_path)
 
+    def _ensure_local_semester_data(self, semester: int) -> None:
+        records_path = self._semester_records_path(semester)
+        index_path = self._semester_index_path(semester)
+
+        if os.path.exists(records_path) and os.path.exists(index_path):
+            return
+
+        if not self.cloud_store:
+            return
+
+        try:
+            payload = self.cloud_store.load_semester_files(semester)
+            if not payload:
+                return
+
+            faiss_bytes, records_bytes = payload
+            with open(index_path, "wb") as f:
+                f.write(faiss_bytes)
+            with open(records_path, "wb") as f:
+                f.write(records_bytes)
+            self._hydration_error_logged_semesters.discard(semester)
+        except Exception as exc:
+            if semester not in self._hydration_error_logged_semesters:
+                print(f"Neon index hydration failed for semester {semester}: {exc}")
+                self._hydration_error_logged_semesters.add(semester)
+
+    def _sync_semester_to_cloud(self, semester: int) -> None:
+        if not self.cloud_store:
+            return
+
+        records_path = self._semester_records_path(semester)
+        index_path = self._semester_index_path(semester)
+
+        if not (os.path.exists(records_path) and os.path.exists(index_path)):
+            return
+
+        try:
+            with open(index_path, "rb") as f:
+                faiss_bytes = f.read()
+            with open(records_path, "rb") as f:
+                records_bytes = f.read()
+
+            self.cloud_store.save_semester_files(semester, faiss_bytes, records_bytes)
+        except Exception as exc:
+            print(f"Neon index sync failed for semester {semester}: {exc}")
+
     @staticmethod
-    def _load(path: str) -> List[dict]:
+    def _load_records(path: str) -> List[dict]:
         if not os.path.exists(path):
             return []
-        with open(path, "r", encoding="utf-8") as f:
-            return json.load(f)
+        with open(path, "rb") as f:
+            return pickle.load(f)
 
     @staticmethod
-    def _save(path: str, data: List[dict]) -> None:
-        with open(path, "w", encoding="utf-8") as f:
-            json.dump(data, f, ensure_ascii=False)
+    def _save_records(path: str, data: List[dict]) -> None:
+        with open(path, "wb") as f:
+            pickle.dump(data, f)