Update app.py

app.py

@@ -2,7 +2,7 @@ import os
 import streamlit as st
 import pandas as pd
 import openai
-import pyodbc
+import sqlite3
 import json
 import numpy as np
 import datetime
@@ -11,9 +11,9 @@ from langchain.llms import OpenAI
 from langchain.schema import Document
 
 # --- CONFIG ---
-AZURE_SQL_CONN_STR = "DRIVER={ODBC Driver 17 for SQL Server};SERVER=sethsrv.database.windows.net;DATABASE=sethdb;UID=seth;PWD=Senth@mil123"
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")  # Or paste your key here
-EMBEDDING_MODEL = "text-embedding-ada-002"    # Or your Azure embedding model
+DB_PATH = "json_vector.db"
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+EMBEDDING_MODEL = "text-embedding-ada-002"
 
 # --- Streamlit State Initialization ---
 if "ingested_batches" not in st.session_state:
@@ -27,8 +27,8 @@ if "modal_content" not in st.session_state:
 if "modal_title" not in st.session_state:
     st.session_state.modal_title = ""
 
-st.set_page_config(page_title="Cumulative JSON Vector Search", layout="wide")
-st.title("LLM-Powered Analytics: Cumulative JSON Vector DB (Azure SQL)")
+st.set_page_config(page_title="Cumulative JSON Vector Search (SQLite)", layout="wide")
+st.title("LLM-Powered Analytics: Cumulative JSON Vector DB (SQLite, Local)")
 
 uploaded_files = st.file_uploader(
     "Upload JSON files in batches (any structure)", type="json", accept_multiple_files=True
@@ -57,18 +57,17 @@ def get_embedding(text):
 
 # --- Ensure DB Table (accumulates all uploads, never deletes old data) ---
 def ensure_table():
-    conn = pyodbc.connect(AZURE_SQL_CONN_STR)
+    conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
     cursor.execute("""
-    IF OBJECT_ID('dbo.json_records', 'U') IS NULL
-        CREATE TABLE json_records (
-            id INT PRIMARY KEY IDENTITY,
-            batch_time DATETIME,
-            source_file NVARCHAR(255),
-            raw_json NVARCHAR(MAX),
-            flat_text NVARCHAR(MAX),
-            embedding VARBINARY(MAX)
-        );
+    CREATE TABLE IF NOT EXISTS json_records (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        batch_time TEXT,
+        source_file TEXT,
+        raw_json TEXT,
+        flat_text TEXT,
+        embedding BLOB
+    )
     """)
     conn.commit()
     conn.close()
@@ -77,7 +76,7 @@ def ensure_table():
 def ingest_json_files(files):
     ensure_table()
     rows = []
-    batch_time = datetime.datetime.utcnow()
+    batch_time = datetime.datetime.utcnow().isoformat()
     for file in files:
         raw = json.load(file)
         source_name = file.name
@@ -85,7 +84,6 @@ def ingest_json_files(files):
         if isinstance(raw, list):
             records = raw
         elif isinstance(raw, dict):
-            # If nested records (like {"people": [...]})
             main_lists = [v for v in raw.values() if isinstance(v, list)]
             if main_lists:
                 records = main_lists[0]
@@ -104,10 +102,10 @@ def ingest_json_files(files):
     st.write(f"Flattened {len(df)} records. Generating embeddings (this may take time, please wait)...")
     df["embedding"] = df["flat_text"].apply(get_embedding)
     # Insert into DB
-    conn = pyodbc.connect(AZURE_SQL_CONN_STR)
+    conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
     for _, row in df.iterrows():
-        emb_bytes = bytearray(np.array(row.embedding, dtype=np.float32).tobytes())
+        emb_bytes = np.array(row.embedding, dtype=np.float32).tobytes()
         cursor.execute("""
             INSERT INTO json_records (batch_time, source_file, raw_json, flat_text, embedding)
             VALUES (?, ?, ?, ?, ?)
@@ -123,12 +121,12 @@ if uploaded_files and st.button("Ingest batch to database"):
 # --- Query entire cumulative DB (ALL past and present records) ---
 def query_vector_db(user_query, top_k=5):
     query_emb = get_embedding(user_query)
-    conn = pyodbc.connect(AZURE_SQL_CONN_STR)
+    conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
     cursor.execute("SELECT id, batch_time, source_file, raw_json, flat_text, embedding FROM json_records")
     results = []
     for row in cursor.fetchall():
-        db_emb = np.frombuffer(row.embedding, dtype=np.float32)
+        db_emb = np.frombuffer(row[5], dtype=np.float32)
         if len(db_emb) != len(query_emb): continue  # Skip malformed
         sim = np.dot(query_emb, db_emb) / (np.linalg.norm(query_emb) * np.linalg.norm(db_emb))
         results.append((sim, row))
@@ -137,35 +135,34 @@ def query_vector_db(user_query, top_k=5):
     docs = []
     for sim, row in results:
         meta = {
-            "id": row.id,
-            "batch_time": str(row.batch_time),
-            "source_file": row.source_file,
+            "id": row[0],
+            "batch_time": str(row[1]),
+            "source_file": row[2],
             "similarity": f"{sim:.4f}",
-            "raw_json": row.raw_json,
+            "raw_json": row[3],
         }
-        docs.append(Document(page_content=row.flat_text, metadata=meta))
+        docs.append(Document(page_content=row[4], metadata=meta))
     return docs
 
 # --- LangChain Retriever ---
-class AzureSQLVectorRetriever:
+class SQLiteVectorRetriever:
     def __init__(self, top_k=5):
         self.top_k = top_k
     def get_relevant_documents(self, query):
         return query_vector_db(query, self.top_k)
 
-llm = OpenAI(model="gpt-4o", openai_api_key=OPENAI_API_KEY, temperature=0)
-retriever = AzureSQLVectorRetriever(top_k=5)
+llm = OpenAI(model="gpt-4.1", openai_api_key=OPENAI_API_KEY, temperature=0)
+retriever = SQLiteVectorRetriever(top_k=5)
 qa_chain = RetrievalQA.from_chain_type(
     llm=llm,
     retriever=retriever,
     return_source_documents=True,
 )
 
-# --- Chat UI & Conversation Loop (preserves your history/modal system) ---
+# --- Chat UI & Conversation Loop (with modal) ---
 st.header("Chat with all accumulated records")
 
 def show_json_links_and_modal():
-    # Scan last result for JSON modal links
     for speaker, msg in reversed(st.session_state.chat_history):
         if speaker == "AI_DOCS":
             docs = msg
@@ -181,7 +178,6 @@ def show_json_links_and_modal():
             if st.button("Close", key="close_modal"):
                 st.session_state.modal_open = False
 
-# --- Chat input ---
 user_input = st.text_input("Ask a question about ALL data (old and new):", key="user_input")
 if st.button("Send") and user_input:
     with st.spinner("Thinking..."):
@@ -190,7 +186,6 @@ if st.button("Send") and user_input:
         st.session_state.chat_history.append(("AI", result['result']))
         st.session_state.chat_history.append(("AI_DOCS", result['source_documents']))
 
-# --- Show conversation ---
 for speaker, msg in st.session_state.chat_history:
     if speaker == "User":
         st.markdown(f"<div style='color: #4F8BF9;'><b>User:</b> {msg}</div>", unsafe_allow_html=True)