Spaces:
Sleeping
Sleeping
File size: 3,321 Bytes
15da411 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | import os
import streamlit as st
from PyPDF2 import PdfReader
from sentence_transformers import SentenceTransformer
import faiss
import numpy as np
from groq import Groq
import requests
from io import BytesIO
# Hardcoded API Key
GROQ_API_KEY = "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"
# Function to download PDF from a URL
def download_pdf_from_url(url):
try:
response = requests.get(url)
response.raise_for_status()
return BytesIO(response.content)
except requests.exceptions.RequestException as e:
st.error(f"Failed to download PDF: {e}")
return None
# Function to extract text from PDF
def extract_text_from_pdf(pdf_file):
reader = PdfReader(pdf_file)
text = ""
for page in reader.pages:
text += page.extract_text()
return text
# Function to split text into chunks
def create_chunks(text, chunk_size=500):
words = text.split()
chunks = [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
return chunks
# Function to create embeddings
def create_embeddings(chunks, model_name='all-MiniLM-L6-v2'):
model = SentenceTransformer(model_name)
embeddings = model.encode(chunks)
return embeddings
# Function to store embeddings in FAISS
def store_embeddings_in_faiss(embeddings):
dimension = embeddings.shape[1]
index = faiss.IndexFlatL2(dimension)
index.add(np.array(embeddings))
return index
# Function to query FAISS index
def query_faiss(index, query_embedding, k=5):
distances, indices = index.search(query_embedding, k)
return indices
# Function to interact with Groq API
def send_query_to_groq(query):
client = Groq(api_key=GROQ_API_KEY)
response = client.chat.completions.create(
messages=[{"role": "user", "content": query}],
model="llama3-8b-8192"
)
return response.choices[0].message.content
# Preload and process PDF links
def preload_pdfs(pdf_links):
st.write("Downloading and processing PDFs...")
all_chunks = []
for url in pdf_links:
pdf_file = download_pdf_from_url(url)
if pdf_file:
text = extract_text_from_pdf(pdf_file)
chunks = create_chunks(text)
all_chunks.extend(chunks)
return all_chunks
# Streamlit UI
def main():
st.title("RAG-based Application")
# Predefined PDF links
pdf_links = [
"https://drive.google.com/uc?id=1hF6exN7tYScy-mxQAP5X9R_200X-ukMB", # Add your links here
# Add more links as needed
]
# Preload PDFs and create embeddings
chunks = preload_pdfs(pdf_links)
embeddings = create_embeddings(chunks)
index = store_embeddings_in_faiss(embeddings)
st.success("All PDFs processed successfully! You can now ask questions.")
# Input for user query
query = st.text_input("Ask your question:")
if query:
st.write("Fetching relevant chunks...")
query_embedding = create_embeddings([query])
relevant_indices = query_faiss(index, query_embedding)
relevant_texts = [chunks[i] for i in relevant_indices[0]]
context = " ".join(relevant_texts)
st.write("Sending query to Groq API...")
response = send_query_to_groq(context)
st.write("Response:", response)
if __name__ == "__main__":
main()
|