Create app.py

app.py

@@ -0,0 +1,113 @@
+import os
+import requests
+import PyPDF2
+import faiss
+import numpy as np
+import streamlit as st
+from transformers import AutoTokenizer, AutoModel
+from groq import Groq
+
+# Download file from Google Drive link
+def download_file_from_drive(url):
+    file_id = url.split("/d/")[1].split("/")[0]
+    download_url = f"https://drive.google.com/uc?id={file_id}&export=download"
+    response = requests.get(download_url)
+    pdf_path = "document.pdf"
+    with open(pdf_path, "wb") as f:
+        f.write(response.content)
+    return pdf_path
+
+# Extract text from PDF
+def extract_text_from_pdf(pdf_path):
+    with open(pdf_path, "rb") as f:
+        reader = PyPDF2.PdfReader(f)
+        text = " ".join(page.extract_text() for page in reader.pages)
+    return text
+
+# Chunk text
+def chunk_text(text, chunk_size=500):
+    words = text.split()
+    chunks = [" ".join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
+    return chunks
+
+# Generate embeddings
+def generate_embeddings(chunks, model_name="sentence-transformers/all-MiniLM-L6-v2"):
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModel.from_pretrained(model_name)
+    embeddings = []
+    for chunk in chunks:
+        inputs = tokenizer(chunk, return_tensors="pt", padding=True, truncation=True)
+        outputs = model(**inputs)
+        embeddings.append(outputs.last_hidden_state.mean(dim=1).detach().numpy())
+    return np.vstack(embeddings)
+
+# Store embeddings in FAISS
+def create_faiss_index(embeddings):
+    dimension = embeddings.shape[1]
+    index = faiss.IndexFlatL2(dimension)
+    index.add(embeddings)
+    return index
+
+# Groq API Integration
+def query_groq_api(query, api_key):
+    client = Groq(api_key=api_key)
+    chat_completion = client.chat.completions.create(
+        messages=[
+            {
+                "role": "user",
+                "content": query,
+            }
+        ],
+        model="llama-3.3-70b-versatile",
+    )
+    return chat_completion.choices[0].message.content
+
+# Streamlit App
+def main():
+    st.title("RAG-based Application")
+    st.sidebar.title("Settings")
+    
+    groq_api_key = st.sidebar.text_input("Enter your Groq API Key", type="password")
+    google_drive_url = st.sidebar.text_input("Enter Google Drive File Link")
+    
+    if st.sidebar.button("Process Document"):
+        st.info("Downloading document...")
+        pdf_path = download_file_from_drive(google_drive_url)
+        st.success("Document downloaded successfully!")
+        
+        st.info("Extracting text...")
+        text = extract_text_from_pdf(pdf_path)
+        st.success("Text extracted successfully!")
+        
+        st.info("Chunking text...")
+        chunks = chunk_text(text)
+        st.success(f"Document chunked into {len(chunks)} chunks.")
+        
+        st.info("Generating embeddings...")
+        embeddings = generate_embeddings(chunks)
+        st.success("Embeddings generated successfully!")
+        
+        st.info("Creating FAISS index...")
+        index = create_faiss_index(embeddings)
+        st.success("FAISS index created successfully!")
+        
+        st.session_state.index = index
+        st.session_state.chunks = chunks
+    
+    if "index" in st.session_state:
+        query = st.text_input("Ask a question:")
+        if st.button("Search"):
+            st.info("Querying FAISS index...")
+            query_embeddings = generate_embeddings([query])
+            distances, indices = st.session_state.index.search(query_embeddings, k=5)
+            relevant_chunks = [st.session_state.chunks[i] for i in indices[0]]
+            st.success("Relevant chunks retrieved!")
+            
+            st.info("Generating answer via Groq API...")
+            context = " ".join(relevant_chunks)
+            answer = query_groq_api(context + "\n" + query, api_key=groq_api_key)
+            st.success("Answer generated!")
+            st.write(answer)
+
+if __name__ == "__main__":
+    main()