Create app.py

app.py

@@ -0,0 +1,205 @@
+import os
+import pandas as pd
+import PyPDF2
+import docx
+from sentence_transformers import SentenceTransformer
+import faiss
+import streamlit as st
+import time
+from groq import Groq
+import re
+
+# Initialize embedding model
+embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
+
+# FAISS setup
+dimension = 384  # Dimension of 'all-MiniLM-L6-v2' embeddings
+index = faiss.IndexFlatL2(dimension)
+document_texts = []  # Store text corresponding to embeddings
+
+# Constants for file handling
+MAX_FILE_SIZE_MB = 100  # 100 MB
+MAX_NUM_FILES = 5
+MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024
+
+# Set up the Groq API client directly with your API key
+api_key = "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"  # Replace with your actual Groq API key
+client = Groq(api_key=api_key)
+
+# Function to get human-readable file size
+def get_human_readable_size(size_in_bytes):
+    if size_in_bytes < 1024:
+        return f"{size_in_bytes} Bytes"
+    elif size_in_bytes < 1024 ** 2:
+        return f"{size_in_bytes / 1024:.2f} KB"
+    elif size_in_bytes < 1024 ** 3:
+        return f"{size_in_bytes / (1024 ** 2):.2f} MB"
+    else:
+        return f"{size_in_bytes / (1024 ** 3):.2f} GB"
+
+# Function to extract text from uploaded files
+def extract_text_from_file(file):
+    text = ""
+    if file.name.endswith(".pdf"):
+        pdf_reader = PyPDF2.PdfReader(file)
+        for page in pdf_reader.pages:
+            text += page.extract_text()
+    elif file.name.endswith(".csv"):
+        df = pd.read_csv(file)
+        text = "\n".join([" ".join(map(str, row)) for row in df.values])
+    elif file.name.endswith(".xlsx") or file.name.endswith(".xls"):
+        df = pd.read_excel(file)
+        text = "\n".join([" ".join(map(str, row)) for row in df.values])
+    elif file.name.endswith(".txt"):
+        text = file.read().decode("utf-8")
+    elif file.name.endswith(".docx"):
+        doc = docx.Document(file)
+        text = "\n".join([p.text for p in doc.paragraphs])
+    else:
+        text = None
+    return text
+
+# Function to split large text into smaller chunks
+def split_text_into_chunks(text, max_chunk_size=500):
+    sentences = text.split(". ")
+    chunks = []
+    chunk = []
+    current_size = 0
+    for sentence in sentences:
+        sentence_size = len(sentence)
+        if current_size + sentence_size <= max_chunk_size:
+            chunk.append(sentence)
+            current_size += sentence_size
+        else:
+            chunks.append(". ".join(chunk))
+            chunk = [sentence]
+            current_size = sentence_size
+    if chunk:
+        chunks.append(". ".join(chunk))
+    return chunks
+
+# Function to add document text to FAISS index
+def add_to_index(text, index, document_texts):
+    chunks = split_text_into_chunks(text)
+    embeddings = embedding_model.encode(chunks, convert_to_numpy=True)
+    index.add(embeddings)
+    document_texts.extend(chunks)
+
+# Function to generate pre-questions based on the document
+def suggest_questions(text):
+    # Example simple questions based on content type
+    if len(text.split()) < 200:
+        return [
+            "Can you summarize the main points?",
+            "What is the main argument or conclusion?",
+            "What is the purpose of this document?"
+        ]
+    else:
+        return [
+            "What are the key takeaways from this document?",
+            "Can you provide a summary of the main sections?",
+            "What are the major findings or conclusions?"
+        ]
+
+# Function to generate answer using Groq
+def generate_answer_with_groq(question, context):
+    # Sending user input question to Groq for response
+    chat_completion = client.chat.completions.create(
+        messages=[{"role": "user", "content": f"Context: {context}\nQuestion: {question}"}],
+        model="gemma2-9b-it",
+    )
+    return chat_completion.choices[0].message.content
+
+# Function to validate user input (basic check for valid text)
+def is_valid_input(query):
+    # Check if the input contains only alphabetic characters, spaces, or common punctuation
+    # This heuristic helps detect typing errors or nonsensical queries
+    query = query.strip()
+    if not query:
+        return False  # Empty input is invalid
+    # Regex to allow letters, spaces, and common punctuation
+    pattern = r"^[A-Za-z0-9\s.,!?'-]*$"
+    if re.match(pattern, query):
+        return True
+    return False
+
+# Handling user feedback
+def handle_feedback(feedback):
+    if feedback:
+        st.write("Thank you for your feedback!")
+
+# Streamlit UI
+st.title("Enhanced Document Q&A with RAG")
+st.sidebar.title("Tips for Better Experience")
+st.sidebar.write("""
+    1. Maximum file size: 100 MB per file.
+    2. You can upload up to 5 files at a time.
+    3. Larger files may take longer to process.
+    4. Please break large files into smaller chunks if necessary.
+    5. Use the pre-generated questions to guide your inquiry.
+""")
+
+feedback = st.sidebar.text_area("Provide feedback to improve your experience:")
+
+# File uploader
+uploaded_files = st.file_uploader(
+    "Upload documents (PDF, CSV, Excel, TXT, DOCX). Max size: 100 MB each.",
+    type=["pdf", "csv", "xlsx", "xls", "txt", "docx"],
+    accept_multiple_files=True,
+)
+
+if uploaded_files:
+    if len(uploaded_files) > MAX_NUM_FILES:
+        st.error(f"Maximum {MAX_NUM_FILES} files can be uploaded at a time.")
+    else:
+        for file in uploaded_files:
+            file_size = file.size
+            human_readable_size = get_human_readable_size(file_size)
+            st.write(f"File: {file.name} | Size: {human_readable_size}")
+            if file_size > MAX_FILE_SIZE_BYTES:
+                st.warning(
+                    f"File '{file.name}' exceeds the {MAX_FILE_SIZE_MB} MB limit. "
+                    "We will automatically break this file into smaller chunks."
+                )
+            with st.spinner(f"Processing {file.name}..."):
+                text = extract_text_from_file(file)
+                if text:
+                    # Automatically break large file into chunks
+                    chunks = split_text_into_chunks(text)
+                    add_to_index(" ".join(chunks), index, document_texts)
+                    st.success(f"Processed {file.name}")
+                else:
+                    st.error(f"Could not process {file.name}. Unsupported format.")
+else:
+    st.warning("No documents uploaded yet. Please upload documents before asking questions.")
+
+# Display user feedback handling
+if feedback:
+    handle_feedback(feedback)
+
+# Input for question
+query = st.text_input("Enter your question:")
+
+# If query is entered and documents are uploaded
+if query:
+    if not document_texts:
+        st.warning("Please upload and process documents before asking questions.")
+    elif not is_valid_input(query):
+        st.error("Please ask a relevant question.")
+    else:
+        # Use Groq to generate a response based on uploaded documents
+        with st.spinner("Generating response..."):
+            response = generate_answer_with_groq(query, " ".join(document_texts))
+            st.write("### Answer:")
+            st.write(response)
+
+        st.write("### Suggested Questions:")
+        questions = suggest_questions(" ".join(document_texts))  # Generate based on full document content
+        for question in questions:
+            st.write(f"- {question}")
+
+# Instructions and reminders if not uploaded_files:
+if not uploaded_files:
+    st.info("You haven't uploaded any documents yet. Please upload documents to start.")
+else:
+    st.info("Enter a question to ask about the uploaded documents.")