Update app.py

app.py

@@ -1,344 +1,218 @@
-import streamlit as st
-import pandas as pd
-import plotly.express as px
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.neural_network import MLPClassifier, MLPRegressor
-from sklearn.cluster import KMeans
-from sklearn.metrics import accuracy_score, r2_score, silhouette_score, confusion_matrix, classification_report, mean_squared_error
-from sklearn.preprocessing import StandardScaler
-from ydata_profiling import ProfileReport
-from streamlit_pandas_profiling import st_profile_report
-from groq import Groq
-from langchain_community.vectorstores import FAISS
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain_community.document_loaders import TextLoader
-from langchain_community.tools.tavily_search import TavilySearchResults
+import gradio as gr
+import groq
 import os
-from dotenv import load_dotenv
 import tempfile
+import uuid
+from dotenv import load_dotenv
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import FAISS
+from langchain.embeddings import HuggingFaceEmbeddings
+import fitz  # PyMuPDF
+import base64
+from PIL import Image
+import io
 
 # Load environment variables
 load_dotenv()
-
-# Initialize Groq client
-client = Groq(api_key=os.getenv("GROQ_API_KEY"))
-
-# Initialize embeddings for FAISS
+client = groq.Client(api_key=os.getenv("GROQ_LEGAL_API_KEY"))
 embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
 
-# Set page config
-st.set_page_config(page_title="Neural-Vision Enhanced", layout="wide")
-
-# Custom CSS matching previous theme
-st.markdown("""
-    <style>
-    :root {
-        --primary-blue: #3B82F6;
-        --dark-blue: #1E40AF;
-        --light-blue: #DBEAFE;
-        --medium-grey: #6B7280;
-        --light-grey: #F3F4F6;
-        --white: #FFFFFF;
-        --border-grey: #E5E7EB;
-    }
-    .stApp {
-        background-color: var(--light-grey);
-        font-family: 'Inter', sans-serif;
-        max-width: 1200px;
-        margin: 0 auto;
-    }
-    .header {
-        background-color: var(--white);
-        border-bottom: 2px solid var(--border-grey);
-        padding: 15px;
-        border-radius: 12px 12px 0 0;
-        box-shadow: 0 2px 4px rgba(0,0,0,0.05);
-        text-align: center;
-    }
-    .header-title {
-        color: var(--dark-blue);
-        font-size: 1.8rem;
-        font-weight: 700;
-        margin: 0;
-    }
-    .header-subtitle {
-        color: var(--medium-grey);
-        font-size: 1rem;
-        margin-top: 5px;
-    }
-    .sidebar .sidebar-content {
-        background-color: var(--white);
-        border-radius: 12px;
-        box-shadow: 0 4px 6px rgba(0,0,0,0.1);
-        padding: 15px;
-    }
-    .chat-container {
-        background-color: var(--white);
-        border-radius: 12px;
-        box-shadow: 0 4px 6px rgba(0,0,0,0.1);
-        padding: 15px;
-        margin-top: 20px;
-    }
-    .user-message {
-        background-color: var(--primary-blue);
-        color: var(--white);
-        border-radius: 18px 18px 4px 18px;
-        padding: 12px 16px;
-        margin-left: auto;
-        max-width: 80%;
-        margin-bottom: 10px;
-    }
-    .bot-message {
-        background-color: var(--light-grey);
-        color: var(--medium-grey);
-        border-radius: 18px 18px 18px 4px;
-        padding: 12px 16px;
-        margin-right: auto;
-        max-width: 80%;
-        margin-bottom: 10px;
-    }
-    </style>
-""", unsafe_allow_html=True)
-
-# Initialize session state
-if 'metrics' not in st.session_state:
-    st.session_state.metrics = {}
-if 'chat_history' not in st.session_state:
-    st.session_state.chat_history = []
-if 'vector_store' not in st.session_state:
-    st.session_state.vector_store = None
-
-# Helper Functions
-def convert_df_to_text(df):
-    text = f"Dataset Summary: {df.shape[0]} rows, {df.shape[1]} columns\n"
-    text += f"Missing Values: {df.isna().sum().sum()}\n"
-    text += "Columns:\n"
-    for col in df.columns:
-        text += f"- {col} ({df[col].dtype}): "
-        if pd.api.types.is_numeric_dtype(df[col]):
-            text += f"Mean={df[col].mean():.2f}, Min={df[col].min()}, Max={df[col].max()}"
-        else:
-            text += f"Unique={df[col].nunique()}, Top={df[col].mode()[0] if not df[col].mode().empty else 'N/A'}"
-        text += f", Missing={df[col].isna().sum()}\n"
-    return text
-
-def create_vector_store(df_text):
-    with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as temp_file:
-        temp_file.write(df_text)
-        temp_path = temp_file.name
-    loader = TextLoader(temp_path)
-    documents = loader.load()
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
-    texts = text_splitter.split_documents(documents)
-    vector_store = FAISS.from_documents(texts, embeddings)
-    os.unlink(temp_path)
-    return vector_store
-
-def get_groq_response(prompt, mode, use_web_search=False):
-    context = ""
-    if st.session_state.vector_store:
-        docs = st.session_state.vector_store.similarity_search(prompt, k=3)
-        context = "\n\nDataset Context:\n" + "\n".join([f"- {doc.page_content}" for doc in docs])
-    
-    if use_web_search:
-        tavily = TavilySearchResults(max_results=3)
-        web_results = tavily.invoke(prompt)
-        context += "\n\nWeb Search Results:\n" + "\n".join([f"- {res['content'][:200]}..." for res in web_results])
-    
-    prompts = {
-        "Legal": "You are a neural network expert specializing in legal data analysis.",
-        "Financial": "You are a neural network expert specializing in financial data analysis.",
-        "Academic": "You are a neural network expert specializing in academic data analysis.",
-        "Technical": "You are a neural network expert specializing in technical data analysis."
-    }
-    system_prompt = prompts.get(mode, "You are a neural network development assistant.") + "\n" + context
-    
-    response = client.chat.completions.create(
-        model="llama3-70b-8192",
-        messages=[
-            {"role": "system", "content": system_prompt},
-            {"role": "user", "content": prompt}
-        ],
-        temperature=0.7,
-        max_tokens=1024
-    )
-    return response.choices[0].message.content
-
-# Visualization Functions
-def plot_confusion_matrix(y_true, y_pred):
-    cm = confusion_matrix(y_true, y_pred)
-    fig = px.imshow(cm, text_auto=True, color_continuous_scale='Blues', title="Confusion Matrix")
-    return fig
-
-def plot_feature_importance(model, X):
-    if hasattr(model, 'feature_importances_'):
-        importance = model.feature_importances_
-    else:
-        importance = np.abs(model.coef_) if hasattr(model, 'coef_') else np.ones(X.shape[1])
-    fig = px.bar(x=X.columns, y=importance, title="Feature Importance")
-    return fig
-
-def plot_residuals(y_true, y_pred):
-    residuals = y_true - y_pred
-    fig = px.scatter(x=y_pred, y=residuals, title="Residual Plot", labels={"x": "Predicted", "y": "Residuals"})
-    return fig
-
-def plot_clusters(X, labels):
-    fig = px.scatter(X, x=X.columns[0], y=X.columns[1], color=labels, title="Cluster Visualization")
-    return fig
-
-# Pages
-def data_upload_page():
-    st.header("📤 Data Upload & Analysis")
-    uploaded_file = st.file_uploader("Upload Dataset", type=["csv"])
-    
-    if uploaded_file:
-        df = pd.read_csv(uploaded_file)
-        st.session_state.df = df
-        st.session_state.vector_store = create_vector_store(convert_df_to_text(df))
-        st.session_state.metrics = {}
-        
-        st.subheader("Dataset Health Check")
-        col1, col2, col3 = st.columns(3)
-        col1.metric("Total Samples", df.shape[0])
-        col2.metric("Features", df.shape[1])
-        col3.metric("Missing Values", df.isna().sum().sum())
+# Directory to store FAISS indexes
+FAISS_INDEX_DIR = "faiss_indexes_academic"
+if not os.path.exists(FAISS_INDEX_DIR):
+    os.makedirs(FAISS_INDEX_DIR)
+
+# Dictionary to store user-specific vectorstores
+user_vectorstores = {}
+
+# Custom CSS for Academic theme
+custom_css = """
+:root {
+    --primary-color: #003366;  /* Deep Blue */
+    --secondary-color: #000080;  /* Navy */
+    --light-background: #F5F5F5;  /* Light Gray */
+    --dark-text: #333333;
+    --white: #FFFFFF;
+    --border-color: #E5E7EB;
+}
+body { background-color: var(--light-background); font-family: 'Inter', sans-serif; }
+.container { max-width: 1200px !important; margin: 0 auto !important; padding: 10px; }
+.header { background-color: var(--white); border-bottom: 2px solid var(--border-color); padding: 15px 0; margin-bottom: 20px; border-radius: 12px 12px 0 0; box-shadow: 0 2px 4px rgba(0,0,0,0.05); }
+.header-title { color: var(--secondary-color); font-size: 1.8rem; font-weight: 700; text-align: center; }
+.header-subtitle { color: var(--dark-text); font-size: 1rem; text-align: center; margin-top: 5px; }
+.chat-container { border-radius: 12px !important; box-shadow: 0 4px 6px rgba(0,0,0,0.1) !important; background-color: var(--white) !important; border: 1px solid var(--border-color) !important; min-height: 500px; }
+.message-user { background-color: var(--primary-color) !important; color: var(--white) !important; border-radius: 18px 18px 4px 18px !important; padding: 12px 16px !important; margin-left: auto !important; max-width: 80% !important; }
+.message-bot { background-color: #F0F0F0 !important; color: var(--dark-text) !important; border-radius: 18px 18px 18px 4px !important; padding: 12px 16px !important; margin-right: auto !important; max-width: 80% !important; }
+.input-area { background-color: var(--white) !important; border-top: 1px solid var(--border-color) !important; padding: 12px !important; border-radius: 0 0 12px 12px !important; }
+.input-box { border: 1px solid var(--border-color) !important; border-radius: 24px !important; padding: 12px 16px !important; box-shadow: 0 2px 4px rgba(0,0,0,0.05) !important; }
+.send-btn { background-color: var(--secondary-color) !important; border-radius: 24px !important; color: var(--white) !important; padding: 10px 20px !important; font-weight: 500 !important; }
+.clear-btn { background-color: #F0F0F0 !important; border: 1px solid var(--border-color) !important; border-radius: 24px !important; color: var(--dark-text) !important; padding: 8px 16px !important; font-weight: 500 !important; }
+.pdf-viewer-container { border-radius: 12px !important; box-shadow: 0 4px 6px rgba(0,0,0,0.1) !important; background-color: var(--white) !important; border: 1px solid var(--border-color) !important; padding: 20px; }
+.pdf-viewer-image { max-width: 100%; height: auto; border: 1px solid var(--border-color); border-radius: 12px; box-shadow: 0 2px 4px rgba(0,0,0,0.05); }
+.stats-box { background-color: #E6E6FA; padding: 10px; border-radius: 8px; margin-top: 10px; }
+"""
+
+# Function to process PDF files (unchanged)
+def process_pdf(pdf_file):
+    if pdf_file is None:
+        return None, "No file uploaded", {"page_images": [], "total_pages": 0, "total_words": 0}
+    try:
+        session_id = str(uuid.uuid4())
+        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:
+            temp_file.write(pdf_file)
+            pdf_path = temp_file.name
         
-        if st.button("Generate Full EDA Report"):
-            with st.spinner("Generating comprehensive analysis..."):
-                profile = ProfileReport(df, explorative=True)
-                st_profile_report(profile)
-
-def model_training_page():
-    st.header("🧠 Neural Network Training Studio")
-    
-    if 'df' not in st.session_state:
-        st.warning("Upload data first!")
-        return
-    
-    df = st.session_state.df
-    problem_type = st.selectbox("Select Problem Type", ["Classification", "Regression", "Clustering"])
-    mode = st.selectbox("Domain Specialization", ["Legal", "Financial", "Academic", "Technical"])
-    
-    if problem_type != "Clustering":
-        target = st.selectbox("Select Target Variable", df.columns)
-        X = df.drop(columns=[target])
-        y = df[target]
-    else:
-        X = df
-        y = None
-    
-    if st.button("Train Neural Network"):
-        with st.spinner("Training in progress..."):
-            X_scaled = StandardScaler().fit_transform(X)
-            X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42) if y is not None else (X_scaled, None, None, None)
-            
-            if problem_type == "Classification":
-                model = MLPClassifier(hidden_layer_sizes=(100, 50), max_iter=500, random_state=42)
-                model.fit(X_train, y_train)
-                y_pred = model.predict(X_test)
-                st.session_state.metrics = {
-                    "Accuracy": accuracy_score(y_test, y_pred),
-                    "Classification Report": classification_report(y_test, y_pred, output_dict=True)
-                }
-            elif problem_type == "Regression":
-                model = MLPRegressor(hidden_layer_sizes=(100, 50), max_iter=500, random_state=42)
-                model.fit(X_train, y_train)
-                y_pred = model.predict(X_test)
-                st.session_state.metrics = {
-                    "R2 Score": r2_score(y_test, y_pred),
-                    "Mean Squared Error": mean_squared_error(y_test, y_pred)
-                }
-            else:  # Clustering
-                model = KMeans(n_clusters=3, random_state=42)
-                labels = model.fit_predict(X_scaled)
-                st.session_state.metrics = {
-                    "Silhouette Score": silhouette_score(X_scaled, labels)
-                }
-            
-            st.session_state.best_model = model
-            st.session_state.X_test = X_test
-            st.session_state.y_test = y_test
-            st.session_state.y_pred = y_pred if y is not None else labels
-            st.session_state.problem_type = problem_type
-            st.success(f"Model trained successfully in {mode} mode!")
-
-def visualization_page():
-    st.header("🔍 Neural Network Evaluation Center")
-    
-    if 'best_model' not in st.session_state:
-        st.warning("Train a model first!")
-        return
+        doc = fitz.open(pdf_path)
+        texts = [page.get_text() for page in doc]
+        page_images = []
+        for page in doc:
+            pix = page.get_pixmap()
+            img_bytes = pix.tobytes("png")
+            img_base64 = base64.b64encode(img_bytes).decode("utf-8")
+            page_images.append(img_base64)
+        total_pages = len(doc)
+        total_words = sum(len(text.split()) for text in texts)
+        doc.close()
+
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+        chunks = text_splitter.create_documents(texts)
+        vectorstore = FAISS.from_documents(chunks, embeddings)
+        index_path = os.path.join(FAISS_INDEX_DIR, session_id)
+        vectorstore.save_local(index_path)
+        user_vectorstores[session_id] = vectorstore
+
+        os.unlink(pdf_path)
+        pdf_state = {"page_images": page_images, "total_pages": total_pages, "total_words": total_words}
+        return session_id, f"✅ Successfully processed {len(chunks)} text chunks from your PDF", pdf_state
+    except Exception as e:
+        if "pdf_path" in locals() and os.path.exists(pdf_path):
+            os.unlink(pdf_path)
+        return None, f"Error processing PDF: {str(e)}", {"page_images": [], "total_pages": 0, "total_words": 0}
+
+# Function to generate chatbot responses with Academic theme
+def generate_response(message, session_id, model_name, history):
+    if not message:
+        return history
+    try:
+        context = ""
+        if session_id and session_id in user_vectorstores:
+            vectorstore = user_vectorstores[session_id]
+            docs = vectorstore.similarity_search(message, k=3)
+            if docs:
+                context = "\n\nRelevant information from uploaded PDF:\n" + "\n".join(f"- {doc.page_content}" for doc in docs)
+        system_prompt = "You are an academic assistant specializing in analyzing research papers, theses, and scholarly articles."
+        if context:
+            system_prompt += " Use the following context to answer the question if relevant: " + context
+        completion = client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": message}
+            ],
+            temperature=0.7,
+            max_tokens=1024
+        )
+        response = completion.choices[0].message.content
+        history.append((message, response))
+        return history
+    except Exception as e:
+        history.append((message, f"Error generating response: {str(e)}"))
+        return history
+
+# Functions to update PDF viewer (unchanged)
+def update_pdf_viewer(pdf_state):
+    if not pdf_state["total_pages"]:
+        return 0, None, "No PDF uploaded yet"
+    try:
+        img_data = base64.b64decode(pdf_state["page_images"][0])
+        img = Image.open(io.BytesIO(img_data))
+        return pdf_state["total_pages"], img, f"**Total Pages:** {pdf_state['total_pages']}\n**Total Words:** {pdf_state['total_words']}"
+    except Exception as e:
+        print(f"Error decoding image: {e}")
+        return 0, None, "Error displaying PDF"
+
+def update_image(page_num, pdf_state):
+    if not pdf_state["total_pages"] or page_num < 1 or page_num > pdf_state["total_pages"]:
+        return None
+    try:
+        img_data = base64.b64decode(pdf_state["page_images"][page_num - 1])
+        img = Image.open(io.BytesIO(img_data))
+        return img
+    except Exception as e:
+        print(f"Error decoding image: {e}")
+        return None
+
+# Gradio interface
+with gr.Blocks(css=custom_css, theme=gr.themes.Soft()) as demo:
+    current_session_id = gr.State(None)
+    pdf_state = gr.State({"page_images": [], "total_pages": 0, "total_words": 0})
+    gr.HTML("""
+    <div class="header">
+        <div class="header-title">Scholar-Vision</div>
+        <div class="header-subtitle">Analyze academic papers with Groq's LLM API.</div>
+    </div>
+    """)
+    with gr.Row(elem_classes="container"):
+        with gr.Column(scale=1, min_width=300):
+            pdf_file = gr.File(label="Upload PDF Document", file_types=[".pdf"], type="binary")
+            upload_button = gr.Button("Process PDF", variant="primary")
+            pdf_status = gr.Markdown("No PDF uploaded yet")
+            model_dropdown = gr.Dropdown(
+                choices=["llama3-70b-8192", "llama3-8b-8192", "mixtral-8x7b-32768", "gemma-7b-it"],
+                value="llama3-70b-8192",
+                label="Select Groq Model"
+            )
+        with gr.Column(scale=2, min_width=600):
+            with gr.Tabs():
+                with gr.TabItem("PDF Viewer"):
+                    with gr.Column(elem_classes="pdf-viewer-container"):
+                        page_slider = gr.Slider(minimum=1, maximum=1, step=1, label="Page Number", value=1)
+                        pdf_image = gr.Image(label="PDF Page", type="pil", elem_classes="pdf-viewer-image")
+                        stats_display = gr.Markdown("No PDF uploaded yet", elem_classes="stats-box")
     
-    st.subheader("Performance Analysis")
-    if st.session_state.problem_type == "Classification":
-        st.plotly_chart(plot_confusion_matrix(st.session_state.y_test, st.session_state.y_pred))
-        st.plotly_chart(plot_feature_importance(st.session_state.best_model, pd.DataFrame(st.session_state.X_test, columns=st.session_state.df.columns[:-1])))
-    elif st.session_state.problem_type == "Regression":
-        st.plotly_chart(plot_residuals(st.session_state.y_test, st.session_state.y_pred))
-        st.plotly_chart(plot_feature_importance(st.session_state.best_model, pd.DataFrame(st.session_state.X_test, columns=st.session_state.df.columns[:-1])))
-    else:  # Clustering
-        st.plotly_chart(plot_clusters(pd.DataFrame(st.session_state.X_test, columns=st.session_state.df.columns), st.session_state.y_pred))
+    with gr.Row(elem_classes="container"):
+        with gr.Column(scale=2, min_width=600):
+            chatbot = gr.Chatbot(height=500, bubble_full_width=False, show_copy_button=True, elem_classes="chat-container")
+            with gr.Row():
+                msg = gr.Textbox(show_label=False, placeholder="Ask about your academic document...", scale=5)
+                send_btn = gr.Button("Send", scale=1)
+            clear_btn = gr.Button("Clear Conversation")
     
-    st.subheader("Metrics")
-    st.write(st.session_state.metrics)
-
-# Chatbot Interface
-def ai_assistant():
-    st.markdown('<div class="chat-container">', unsafe_allow_html=True)
-    st.subheader("🧠 Neural Insight Assistant (RAG + Web Search)")
+    # Event Handlers (unchanged)
+    upload_button.click(
+        process_pdf,
+        inputs=[pdf_file],
+        outputs=[current_session_id, pdf_status, pdf_state]
+    ).then(
+        update_pdf_viewer,
+        inputs=[pdf_state],
+        outputs=[page_slider, pdf_image, stats_display]
+    )
     
-    use_web_search = st.checkbox("Enable Tavily Web Search", value=False)
-    mode = st.selectbox("Domain Mode", ["Legal", "Financial", "Academic", "Technical"], key="chat_mode")
+    msg.submit(
+        generate_response,
+        inputs=[msg, current_session_id, model_dropdown, chatbot],
+        outputs=[chatbot]
+    ).then(lambda: "", None, [msg])
     
-    for msg in st.session_state.chat_history:
-        with st.chat_message(msg["role"]):
-            st.markdown(f'<div class="{msg["role"]}-message">{msg["content"]}</div>', unsafe_allow_html=True)
+    send_btn.click(
+        generate_response,
+        inputs=[msg, current_session_id, model_dropdown, chatbot],
+        outputs=[chatbot]
+    ).then(lambda: "", None, [msg])
     
-    if prompt := st.chat_input("Ask about data, models, or web insights..."):
-        st.session_state.chat_history.append({"role": "user", "content": prompt})
-        with st.chat_message("user"):
-            st.markdown(f'<div class="user-message">{prompt}</div>', unsafe_allow_html=True)
-        
-        with st.spinner("Processing..."):
-            response = get_groq_response(prompt, mode, use_web_search)
-            st.session_state.chat_history.append({"role": "assistant", "content": response})
-        
-        with st.chat_message("assistant"):
-            st.markdown(f'<div class="bot-message">{response}</div>', unsafe_allow_html=True)
+    clear_btn.click(
+        lambda: ([], None, "No PDF uploaded yet", {"page_images": [], "total_pages": 0, "total_words": 0}, 0, None, "No PDF uploaded yet"),
+        None,
+        [chatbot, current_session_id, pdf_status, pdf_state, page_slider, pdf_image, stats_display]
+    )
     
-    st.markdown('</div>', unsafe_allow_html=True)
-
-# Main App Layout
-st.markdown("""
-    <div class="header">
-        <h1 class="header-title">Neural-Vision Enhanced</h1>
-        <div class="header-subtitle">Neural Network Development for Domain-Specialized Analysis</div>
-    </div>
-""", unsafe_allow_html=True)
-
-with st.sidebar:
-    st.title("🔮 Neural-Vision Enhanced")
-    page = st.selectbox("Navigation", [
-        "Data Upload & Analysis",
-        "Neural Network Training Studio",
-        "Neural Network Evaluation Center"
-    ])
-    st.session_state.active_page = page
-    st.markdown("---")
-    st.markdown("**Environment Setup**")
-    os.environ["TAVILY_API_KEY"] = st.text_input("Tavily API Key", type="password", help="For web search functionality")
-    st.markdown("---")
-    st.markdown("v5.0 | © 2025 Neural-Vision")
-
-# Page Routing
-if "Data Upload & Analysis" in page:
-    data_upload_page()
-elif "Neural Network Training Studio" in page:
-    model_training_page()
-else:
-    visualization_page()
+    page_slider.change(
+        update_image,
+        inputs=[page_slider, pdf_state],
+        outputs=[pdf_image]
+    )
 
-ai_assistant()
+# Launch the app
+if __name__ == "__main__":
+    demo.launch()