Update app.py

app.py

@@ -1,23 +1,25 @@
-import streamlit as st
 import os
+import streamlit as st
+import pandas as pd
+import openai
+import pyodbc
 import json
-import requests
-import traceback
-import difflib
-
-COMMON_FEMALE_NAMES = {
-    "alice", "mary", "lisa", "jane", "emily", "sophia", "emma", "olivia",
-    "ava", "mia", "isabella", "charlotte", "amelia", "harper", "abigail"
-}
-
-if "json_data" not in st.session_state:
-    st.session_state.json_data = {}
-if "messages" not in st.session_state:
-    st.session_state.messages = []
-if "files_loaded" not in st.session_state:
-    st.session_state.files_loaded = False
-if "temp_input" not in st.session_state:
-    st.session_state.temp_input = ""
+import numpy as np
+import datetime
+from langchain.chains import RetrievalQA
+from langchain.llms import OpenAI
+from langchain.schema import Document
+
+# --- CONFIG ---
+AZURE_SQL_CONN_STR = "DRIVER={ODBC Driver 17 for SQL Server};SERVER=<server>.database.windows.net;DATABASE=<db>;UID=<user>;PWD=<password>"
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")  # Or paste your key here
+EMBEDDING_MODEL = "text-embedding-ada-002"    # Or your Azure embedding model
+
+# --- Streamlit State Initialization ---
+if "ingested_batches" not in st.session_state:
+    st.session_state.ingested_batches = 0
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = []
 if "modal_open" not in st.session_state:
     st.session_state.modal_open = False
 if "modal_content" not in st.session_state:
@@ -25,420 +27,179 @@ if "modal_content" not in st.session_state:
 if "modal_title" not in st.session_state:
     st.session_state.modal_title = ""
 
-st.set_page_config(page_title="Chat with Your JSONs", layout="wide")
-st.title("Chat with Your JSON Files (OpenAI function-calling, No LangChain)")
+st.set_page_config(page_title="Cumulative JSON Vector Search", layout="wide")
+st.title("LLM-Powered Analytics: Cumulative JSON Vector DB (Azure SQL)")
 
-uploaded_files = st.sidebar.file_uploader(
-    "Choose one or more JSON files", type="json", accept_multiple_files=True
+uploaded_files = st.file_uploader(
+    "Upload JSON files in batches (any structure)", type="json", accept_multiple_files=True
 )
-if uploaded_files and not st.session_state.files_loaded:
-    st.session_state.json_data.clear()
-    for f in uploaded_files:
-        try:
-            content = json.load(f)
-            st.session_state.json_data[f.name] = content
-            st.sidebar.success(f"Loaded: {f.name}")
-        except Exception as e:
-            st.sidebar.error(f"Error reading {f.name}: {e}")
-    st.session_state.files_loaded = True
-    st.session_state.messages = []
-elif not uploaded_files:
-    st.session_state.json_data.clear()
-    st.session_state.files_loaded = False
-
-def normalize(s):
-    return ' '.join(str(s).lower().replace("_", " ").replace("-", " ").replace(".", " ").split())
-
-def is_fuzzy_match(a, b, threshold=0.7):
-    ratio = difflib.SequenceMatcher(None, a, b).ratio()
-    return ratio >= threshold or a in b or b in a
-
-def search_all_jsons(key, value):
-    matches = []
-    value_norm = normalize(value)
-    for file_name, data in st.session_state.json_data.items():
-        def recursive_search(obj):
-            if isinstance(obj, dict):
-                for k, v in obj.items():
-                    if normalize(k) == normalize(key):
-                        if isinstance(v, (str, int, float, bool)) and is_fuzzy_match(value_norm, normalize(v)):
-                            matches.append({
-                                "file": file_name,
-                                "key": k,
-                                "value": v,
-                                "record": obj
-                            })
-                    recursive_search(v)
-            elif isinstance(obj, list):
-                for item in obj:
-                    recursive_search(item)
-        recursive_search(data)
-    return matches
-
-def fuzzy_value_search(value):
-    matches = []
-    value_norm = normalize(value)
-    for file_name, data in st.session_state.json_data.items():
-        def recursive_search(obj):
-            if isinstance(obj, dict):
-                for k, v in obj.items():
-                    if isinstance(v, (str, int, float, bool)) and is_fuzzy_match(value_norm, normalize(v)):
-                        matches.append({
-                            "file": file_name,
-                            "key": k,
-                            "value": v,
-                            "record": obj
-                        })
-                    recursive_search(v)
-            elif isinstance(obj, list):
-                for item in obj:
-                    recursive_search(item)
-        recursive_search(data)
-    return matches
 
-def list_keys(file_name):
-    try:
-        data = st.session_state.json_data[file_name]
-        if isinstance(data, dict):
-            return list(data.keys())
-        elif isinstance(data, list) and data and isinstance(data[0], dict):
-            return list(data[0].keys())
+# --- Helper: Flatten any unstructured JSON (handles dict, list, nested, various keys) ---
+def flatten_json_obj(obj, parent_key="", sep="."):
+    items = {}
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            new_key = f"{parent_key}{sep}{k}" if parent_key else k
+            items.update(flatten_json_obj(v, new_key, sep=sep))
+    elif isinstance(obj, list):
+        for i, v in enumerate(obj):
+            new_key = f"{parent_key}{sep}{i}" if parent_key else str(i)
+            items.update(flatten_json_obj(v, new_key, sep=sep))
+    else:
+        items[parent_key] = obj
+    return items
+
+# --- Embedding function ---
+def get_embedding(text):
+    openai.api_key = OPENAI_API_KEY
+    resp = openai.Embedding.create(input=text, model=EMBEDDING_MODEL)
+    return resp['data'][0]['embedding']
+
+# --- Ensure DB Table (accumulates all uploads, never deletes old data) ---
+def ensure_table():
+    conn = pyodbc.connect(AZURE_SQL_CONN_STR)
+    cursor = conn.cursor()
+    cursor.execute("""
+    IF OBJECT_ID('dbo.json_records', 'U') IS NULL
+        CREATE TABLE json_records (
+            id INT PRIMARY KEY IDENTITY,
+            batch_time DATETIME,
+            source_file NVARCHAR(255),
+            raw_json NVARCHAR(MAX),
+            flat_text NVARCHAR(MAX),
+            embedding VARBINARY(MAX)
+        );
+    """)
+    conn.commit()
+    conn.close()
+
+# --- Ingest and accumulate uploaded files ---
+def ingest_json_files(files):
+    ensure_table()
+    rows = []
+    batch_time = datetime.datetime.utcnow()
+    for file in files:
+        raw = json.load(file)
+        source_name = file.name
+        # Handle top-level list or dict
+        if isinstance(raw, list):
+            records = raw
+        elif isinstance(raw, dict):
+            # If nested records (like {"people": [...]})
+            main_lists = [v for v in raw.values() if isinstance(v, list)]
+            if main_lists:
+                records = main_lists[0]
+            else:
+                records = [raw]
         else:
-            return []
-    except Exception as e:
-        return {"error": str(e)}
-
-def count_key_occurrences(file_name, key):
-    try:
-        data = st.session_state.json_data[file_name]
-        count = 0
-        def recursive(obj):
-            nonlocal count
-            if isinstance(obj, dict):
-                for k, v in obj.items():
-                    if normalize(k) == normalize(key):
-                        count += 1
-                    recursive(v)
-            elif isinstance(obj, list):
-                for item in obj:
-                    recursive(item)
-        recursive(data)
-        return count
-    except Exception as e:
-        return {"error": str(e)}
-
-def find_in_arrays(key, value, return_count=True):
-    matches = []
-    count = 0
-    for file_name, data in st.session_state.json_data.items():
-        def recursive(obj):
-            nonlocal count
-            if isinstance(obj, list):
-                for item in obj:
-                    if isinstance(item, dict):
-                        for k, v in item.items():
-                            if normalize(k) == normalize(key) and normalize(str(v)) == normalize(str(value)):
-                                matches.append({
-                                    "file": file_name,
-                                    "item": item,
-                                    "array_path": key,
-                                    "record": item  # show the item dict itself for popup
-                                })
-                                count += 1
-                    recursive(item)
-            elif isinstance(obj, dict):
-                for v in obj.values():
-                    recursive(v)
-        recursive(data)
-    return count if return_count else matches
-
-def sum_field_by_name(name, field="amount"):
-    total = 0
-    details = []
-    name_norm = normalize(name)
-    for file_name, data in st.session_state.json_data.items():
-        def recursive(obj):
-            nonlocal total
-            if isinstance(obj, dict):
-                for k, v in obj.items():
-                    if isinstance(v, (str, int, float, bool)) and is_fuzzy_match(name_norm, normalize(v)):
-                        if field in obj:
-                            try:
-                                amt = float(obj[field])
-                                total += amt
-                                details.append({"file": file_name, "name_match": v, "amount": amt, "record": obj})
-                            except Exception:
-                                pass
-                    recursive(v)
-            elif isinstance(obj, list):
-                for item in obj:
-                    recursive(item)
-        recursive(data)
-    return {"total": total, "matches": details}
-
-def count_female_names():
-    count = 0
-    names = []
-    for file_name, data in st.session_state.json_data.items():
-        def recursive(obj):
-            nonlocal count
-            if isinstance(obj, dict):
-                for k, v in obj.items():
-                    if k.lower() in {"name", "fullName", "firstName"}:
-                        first_name = str(v).split()[0].lower()
-                        if first_name in COMMON_FEMALE_NAMES:
-                            count += 1
-                            names.append({"file": file_name, "name": v, "record": obj})
-                    recursive(v)
-            elif isinstance(obj, list):
-                for item in obj:
-                    recursive(item)
-        recursive(data)
-    return {"count": count, "names": names}
-
-function_schema = [
-    {
-        "name": "search_all_jsons",
-        "description": "Recursively search all uploaded JSONs for all records where a key matches a value (fuzzy, any type).",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "key": {"type": "string"},
-                "value": {"type": "string"}
-            },
-            "required": ["key", "value"]
-        }
-    },
-    {
-        "name": "fuzzy_value_search",
-        "description": "Search all uploaded JSONs for any record with a field value matching (fuzzy, all types).",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "value": {"type": "string"}
-            },
-            "required": ["value"]
-        }
-    },
-    {
-        "name": "list_keys",
-        "description": "List top-level keys in a given JSON file.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "file_name": {"type": "string"}
-            },
-            "required": ["file_name"]
+            records = [raw]
+        for rec in records:
+            flat = flatten_json_obj(rec)
+            flat_text = "; ".join([f"{k}: {v}" for k, v in flat.items()])
+            rows.append((batch_time, source_name, json.dumps(rec), flat_text))
+    if not rows:
+        st.warning("No records found in uploaded files!")
+        return
+    df = pd.DataFrame(rows, columns=["batch_time", "source_file", "raw_json", "flat_text"])
+    st.write(f"Flattened {len(df)} records. Generating embeddings (this may take time, please wait)...")
+    df["embedding"] = df["flat_text"].apply(get_embedding)
+    # Insert into DB
+    conn = pyodbc.connect(AZURE_SQL_CONN_STR)
+    cursor = conn.cursor()
+    for _, row in df.iterrows():
+        emb_bytes = bytearray(np.array(row.embedding, dtype=np.float32).tobytes())
+        cursor.execute("""
+            INSERT INTO json_records (batch_time, source_file, raw_json, flat_text, embedding)
+            VALUES (?, ?, ?, ?, ?)
+        """, row.batch_time, row.source_file, row.raw_json, row.flat_text, emb_bytes)
+    conn.commit()
+    conn.close()
+    st.success(f"Ingested and indexed {len(df)} new records!")
+    st.session_state.ingested_batches += 1
+
+if uploaded_files and st.button("Ingest batch to database"):
+    ingest_json_files(uploaded_files)
+
+# --- Query entire cumulative DB (ALL past and present records) ---
+def query_vector_db(user_query, top_k=5):
+    query_emb = get_embedding(user_query)
+    conn = pyodbc.connect(AZURE_SQL_CONN_STR)
+    cursor = conn.cursor()
+    cursor.execute("SELECT id, batch_time, source_file, raw_json, flat_text, embedding FROM json_records")
+    results = []
+    for row in cursor.fetchall():
+        db_emb = np.frombuffer(row.embedding, dtype=np.float32)
+        if len(db_emb) != len(query_emb): continue  # Skip malformed
+        sim = np.dot(query_emb, db_emb) / (np.linalg.norm(query_emb) * np.linalg.norm(db_emb))
+        results.append((sim, row))
+    conn.close()
+    results = sorted(results, reverse=True)[:top_k]
+    docs = []
+    for sim, row in results:
+        meta = {
+            "id": row.id,
+            "batch_time": str(row.batch_time),
+            "source_file": row.source_file,
+            "similarity": f"{sim:.4f}",
+            "raw_json": row.raw_json,
         }
-    },
-    {
-        "name": "count_key_occurrences",
-        "description": "Count number of times a key appears in a file.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "file_name": {"type": "string"},
-                "key": {"type": "string"}
-            },
-            "required": ["file_name", "key"]
-        }
-    },
-    {
-        "name": "find_in_arrays",
-        "description": "Find/count all objects in any arrays/lists where key equals value (e.g. done:true for completed tasks).",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "key": {"type": "string", "description": "The key to search for, e.g., 'done'"},
-                "value": {"type": "string", "description": "The value to match, e.g., 'true' or 'false'"},
-                "return_count": {"type": "boolean", "description": "Return the count (true) or matching records (false)."}
-            },
-            "required": ["key", "value"]
-        }
-    },
-    {
-        "name": "sum_field_by_name",
-        "description": "Sum a field (e.g. amount) for any record containing a name/email/identifier. Returns total and breakdown.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "name": {"type": "string", "description": "Name or identifier to match"},
-                "field": {"type": "string", "description": "The numeric field to sum, e.g. 'amount'"},
-            },
-            "required": ["name", "field"]
-        }
-    },
-    {
-        "name": "count_female_names",
-        "description": "Count the number of common female names based on a preset list.",
-        "parameters": {
-            "type": "object",
-            "properties": {},
-        }
-    }
-]
-
-system_message = {
-    "role": "system",
-    "content": (
-        "You are a JSON data assistant. Use the functions provided to answer the user's question. "
-        "If the user asks for the number or details of items in a list/array (e.g., completed tasks), use 'find_in_arrays'. "
-        "If the user asks about the sum/total of a field for a name or identifier, use 'sum_field_by_name'. "
-        "If the user asks about female names, use 'count_female_names'. "
-        "If the user's query does not mention a key, use 'fuzzy_value_search' to match on any value. "
-        "If a key is mentioned (like 'apps_installed'), use 'search_all_jsons' for that key and the value. "
-        "You may use 'list_keys' to help discover the file structure if needed. "
-        "Always give a direct answer from the data if possible."
-    )
-}
-
-st.markdown("### Ask any question about your data, just like ChatGPT.")
+        docs.append(Document(page_content=row.flat_text, metadata=meta))
+    return docs
+
+# --- LangChain Retriever ---
+class AzureSQLVectorRetriever:
+    def __init__(self, top_k=5):
+        self.top_k = top_k
+    def get_relevant_documents(self, query):
+        return query_vector_db(query, self.top_k)
+
+llm = OpenAI(model="gpt-4o", openai_api_key=OPENAI_API_KEY, temperature=0)
+retriever = AzureSQLVectorRetriever(top_k=5)
+qa_chain = RetrievalQA.from_chain_type(
+    llm=llm,
+    retriever=retriever,
+    return_source_documents=True,
+)
 
-for msg in st.session_state.messages:
-    if msg["role"] == "user":
-        st.markdown(f"<div style='color: #4F8BF9;'><b>User:</b> {msg['content']}</div>", unsafe_allow_html=True)
-    elif msg["role"] == "assistant":
-        st.markdown(f"<div style='color: #1C6E4C;'><b>Agent:</b> {msg['content']}</div>", unsafe_allow_html=True)
-    elif msg["role"] == "function":
-        st.markdown(f"<details><summary><b>Function '{msg['name']}' output:</b></summary><pre>{msg['content']}</pre></details>", unsafe_allow_html=True)
+# --- Chat UI & Conversation Loop (preserves your history/modal system) ---
+st.header("Chat with all accumulated records")
 
-# --- JSON MODAL POPUP ---
 def show_json_links_and_modal():
-    # Find last function message
-    for msg in reversed(st.session_state.messages):
-        if msg.get("role") == "function":
-            func_name = msg.get("name")
-            try:
-                content = json.loads(msg["content"])
-            except Exception:
-                content = None
-            links_shown = False
-            if isinstance(content, list):
-                for idx, match in enumerate(content):
-                    if isinstance(match, dict) and "record" in match:
-                        if st.button(f"View JSON: {match.get('file', 'unknown')} record #{idx+1}", key=f"modal_{func_name}_{idx}"):
-                            st.session_state.modal_open = True
-                            st.session_state.modal_content = json.dumps(match["record"], indent=2)
-                            st.session_state.modal_title = f"{match.get('file', 'unknown')} record #{idx+1}"
-                        links_shown = True
-            elif isinstance(content, dict):
-                # For dicts with matches
-                if "matches" in content and isinstance(content["matches"], list):
-                    for idx, match in enumerate(content["matches"]):
-                        if isinstance(match, dict) and "record" in match:
-                            if st.button(f"View JSON: {match.get('file', 'unknown')} record #{idx+1}", key=f"modal_{func_name}_matches_{idx}"):
-                                st.session_state.modal_open = True
-                                st.session_state.modal_content = json.dumps(match["record"], indent=2)
-                                st.session_state.modal_title = f"{match.get('file', 'unknown')} record #{idx+1}"
-                            links_shown = True
-                if "names" in content and isinstance(content["names"], list):
-                    for idx, match in enumerate(content["names"]):
-                        if isinstance(match, dict) and "record" in match:
-                            if st.button(f"View JSON: {match.get('file', 'unknown')} record #{idx+1}", key=f"modal_{func_name}_names_{idx}"):
-                                st.session_state.modal_open = True
-                                st.session_state.modal_content = json.dumps(match["record"], indent=2)
-                                st.session_state.modal_title = f"{match.get('file', 'unknown')} record #{idx+1}"
-                            links_shown = True
-            if links_shown:
-                break
-
-    # Modal popup UI using st.expander as a modal hack
+    # Scan last result for JSON modal links
+    for speaker, msg in reversed(st.session_state.chat_history):
+        if speaker == "AI_DOCS":
+            docs = msg
+            for idx, doc in enumerate(docs):
+                if st.button(f"View JSON: {doc.metadata['source_file']} (#{doc.metadata['id']})", key=f"modal_{idx}"):
+                    st.session_state.modal_open = True
+                    st.session_state.modal_content = json.dumps(json.loads(doc.metadata["raw_json"]), indent=2)
+                    st.session_state.modal_title = f"{doc.metadata['source_file']} (#{doc.metadata['id']})"
+            break
     if st.session_state.modal_open:
         with st.expander(f"JSON Record: {st.session_state.modal_title}", expanded=True):
             st.code(st.session_state.modal_content, language="json")
             if st.button("Close", key="close_modal"):
                 st.session_state.modal_open = False
 
-show_json_links_and_modal()
-
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
-HEADERS = {
-    "Authorization": f"Bearer {OPENAI_API_KEY}",
-    "Content-Type": "application/json",
-}
-
-def send_message():
-    try:
-        user_input = st.session_state.temp_input
-        if user_input.strip():
-            st.session_state.messages.append({"role": "user", "content": user_input})
-            chat_messages = [system_message] + st.session_state.messages[-10:]
-            chat_resp = requests.post(
-                "https://api.openai.com/v1/chat/completions",
-                headers=HEADERS,
-                json={
-                    "model": "gpt-4o",
-                    "messages": chat_messages,
-                    "functions": function_schema,
-                    "function_call": "auto",
-                    "temperature": 0,
-                    "max_tokens": 1200,
-                },
-                timeout=60,
-            )
-            chat_resp.raise_for_status()
-            response_json = chat_resp.json()
-            msg = response_json["choices"][0]["message"]
+# --- Chat input ---
+user_input = st.text_input("Ask a question about ALL data (old and new):", key="user_input")
+if st.button("Send") and user_input:
+    with st.spinner("Thinking..."):
+        result = qa_chain(user_input)
+        st.session_state.chat_history.append(("User", user_input))
+        st.session_state.chat_history.append(("AI", result['result']))
+        st.session_state.chat_history.append(("AI_DOCS", result['source_documents']))
+
+# --- Show conversation ---
+for speaker, msg in st.session_state.chat_history:
+    if speaker == "User":
+        st.markdown(f"<div style='color: #4F8BF9;'><b>User:</b> {msg}</div>", unsafe_allow_html=True)
+    elif speaker == "AI":
+        st.markdown(f"<div style='color: #1C6E4C;'><b>Agent:</b> {msg}</div>", unsafe_allow_html=True)
 
-            if msg.get("function_call"):
-                func_name = msg["function_call"]["name"]
-                args_json = msg["function_call"]["arguments"]
-                args = json.loads(args_json)
-
-                if func_name == "search_all_jsons":
-                    result = search_all_jsons(args.get("key"), args.get("value"))
-                elif func_name == "fuzzy_value_search":
-                    result = fuzzy_value_search(args.get("value"))
-                elif func_name == "list_keys":
-                    result = list_keys(args.get("file_name"))
-                elif func_name == "count_key_occurrences":
-                    result = count_key_occurrences(args.get("file_name"), args.get("key"))
-                elif func_name == "find_in_arrays":
-                    result = find_in_arrays(
-                        args.get("key"),
-                        args.get("value"),
-                        args.get("return_count", True)
-                    )
-                elif func_name == "sum_field_by_name":
-                    result = sum_field_by_name(
-                        args.get("name"),
-                        args.get("field", "amount")
-                    )
-                elif func_name == "count_female_names":
-                    result = count_female_names()
-                else:
-                    result = {"error": f"Unknown function: {func_name}"}
-
-                st.session_state.messages.append({
-                    "role": "function",
-                    "name": func_name,
-                    "content": json.dumps(result, indent=2),
-                })
+show_json_links_and_modal()
 
-                followup_messages = chat_messages + [
-                    {"role": "function", "name": func_name, "content": json.dumps(result, indent=2)}
-                ]
-                final_resp = requests.post(
-                    "https://api.openai.com/v1/chat/completions",
-                    headers=HEADERS,
-                    json={
-                        "model": "gpt-4o",
-                        "messages": followup_messages,
-                        "temperature": 0,
-                        "max_tokens": 1200,
-                    },
-                    timeout=60,
-                )
-                final_resp.raise_for_status()
-                final_json = final_resp.json()
-                answer = final_json["choices"][0]["message"]["content"]
-                st.session_state.messages.append({"role": "assistant", "content": answer})
-        st.session_state.temp_input = ""
-    except Exception as e:
-        st.error("Exception: " + str(e))
-        st.code(traceback.format_exc())
+if st.button("Clear chat"):
+    st.session_state.chat_history = []
 
-if st.session_state.json_data:
-    st.text_input("Your message:", key="temp_input", on_change=send_message)
-else:
-    st.info("Please upload at least one JSON file to start chatting.")
+st.info(f"Batches ingested so far (this session): {st.session_state.ingested_batches}")