Upload 2 files
Browse files- app.py +108 -0
- requirements.txt +9 -0
app.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =======================================
|
| 2 |
+
# π RAG App β Gemini + Local Embeddings
|
| 3 |
+
# =======================================
|
| 4 |
+
|
| 5 |
+
#!pip install google-generativeai sentence-transformers chromadb beautifulsoup4 PyPDF2 gradio
|
| 6 |
+
|
| 7 |
+
import os, textwrap, re
|
| 8 |
+
import google.generativeai as genai
|
| 9 |
+
from sentence_transformers import SentenceTransformer
|
| 10 |
+
from bs4 import BeautifulSoup
|
| 11 |
+
import requests
|
| 12 |
+
import chromadb
|
| 13 |
+
from PyPDF2 import PdfReader
|
| 14 |
+
import gradio as gr
|
| 15 |
+
|
| 16 |
+
# ======================
|
| 17 |
+
# πΉ API Key Setup
|
| 18 |
+
# ======================
|
| 19 |
+
genai.configure(api_key="AIzaSyDr2X5N-hHt9EqUNy7JCm58aG1FpeGVpgs") # apni key yahan daalo
|
| 20 |
+
|
| 21 |
+
MODEL = 'gemini-2.5-flash'
|
| 22 |
+
embedder = SentenceTransformer('all-MiniLM-L6-v2') # local free embedding model
|
| 23 |
+
chroma_client = chromadb.Client()
|
| 24 |
+
collection = chroma_client.create_collection(name="rag_store")
|
| 25 |
+
|
| 26 |
+
# ======================
|
| 27 |
+
# πΉ Helper Functions
|
| 28 |
+
# ======================
|
| 29 |
+
|
| 30 |
+
def chunk_text(text, size=1000, overlap=100):
|
| 31 |
+
chunks = []
|
| 32 |
+
for i in range(0, len(text), size - overlap):
|
| 33 |
+
chunks.append(text[i:i+size])
|
| 34 |
+
return chunks
|
| 35 |
+
|
| 36 |
+
def clean_text(text):
|
| 37 |
+
text = re.sub(r'\s+', ' ', text)
|
| 38 |
+
return text.strip()
|
| 39 |
+
|
| 40 |
+
def ingest_source(source, from_url=True):
|
| 41 |
+
"""
|
| 42 |
+
β
Web URL ya PDF se text nikaalo aur Chroma me store karo
|
| 43 |
+
"""
|
| 44 |
+
if from_url:
|
| 45 |
+
html = requests.get(source).text
|
| 46 |
+
soup = BeautifulSoup(html, "html.parser")
|
| 47 |
+
text = clean_text(soup.get_text())
|
| 48 |
+
else:
|
| 49 |
+
reader = PdfReader(source)
|
| 50 |
+
text = " ".join([page.extract_text() for page in reader.pages])
|
| 51 |
+
|
| 52 |
+
chunks = chunk_text(text)
|
| 53 |
+
embeddings = embedder.encode(chunks).tolist()
|
| 54 |
+
|
| 55 |
+
for i, emb in enumerate(embeddings):
|
| 56 |
+
collection.add(ids=[f"doc_{i}"], embeddings=[emb], documents=[chunks[i]])
|
| 57 |
+
print(f"β
Ingested {len(chunks)} chunks into Chroma DB")
|
| 58 |
+
|
| 59 |
+
def rag_query(query):
|
| 60 |
+
"""
|
| 61 |
+
β
Query kare aur best-matched chunks Gemini ko dekar answer banaye
|
| 62 |
+
"""
|
| 63 |
+
q_emb = embedder.encode([query]).tolist()
|
| 64 |
+
results = collection.query(query_embeddings=q_emb, n_results=4)
|
| 65 |
+
context = " ".join(results['documents'][0])
|
| 66 |
+
|
| 67 |
+
prompt = f"""
|
| 68 |
+
You are an AI assistant. Use the context below to answer clearly:
|
| 69 |
+
Context: {context}
|
| 70 |
+
Question: {query}
|
| 71 |
+
Answer:
|
| 72 |
+
"""
|
| 73 |
+
response = genai.GenerativeModel(MODEL).generate_content(prompt)
|
| 74 |
+
return textwrap.fill(response.text, width=100)
|
| 75 |
+
|
| 76 |
+
# ======================
|
| 77 |
+
# πΉ Gradio UI
|
| 78 |
+
# ======================
|
| 79 |
+
def web_ingest_ui(url):
|
| 80 |
+
ingest_source(url, from_url=True)
|
| 81 |
+
return f"β
Website data added: {url}"
|
| 82 |
+
|
| 83 |
+
def pdf_ingest_ui(file):
|
| 84 |
+
ingest_source(file.name, from_url=False)
|
| 85 |
+
return f"β
PDF data added: {file.name}"
|
| 86 |
+
|
| 87 |
+
with gr.Blocks(theme=gr.themes.Soft(primary_hue="teal")) as demo:
|
| 88 |
+
gr.Markdown("## π€ RAG App (Gemini + Local Embeddings)")
|
| 89 |
+
|
| 90 |
+
with gr.Tab("π Ingest Website"):
|
| 91 |
+
url_in = gr.Textbox(label="Enter Website URL")
|
| 92 |
+
url_btn = gr.Button("Ingest Website")
|
| 93 |
+
url_out = gr.Textbox(label="Status")
|
| 94 |
+
url_btn.click(fn=web_ingest_ui, inputs=url_in, outputs=url_out)
|
| 95 |
+
|
| 96 |
+
with gr.Tab("π Ingest PDF"):
|
| 97 |
+
pdf_in = gr.File(label="Upload PDF")
|
| 98 |
+
pdf_btn = gr.Button("Ingest PDF")
|
| 99 |
+
pdf_out = gr.Textbox(label="Status")
|
| 100 |
+
pdf_btn.click(fn=pdf_ingest_ui, inputs=pdf_in, outputs=pdf_out)
|
| 101 |
+
|
| 102 |
+
with gr.Tab("π¬ Ask Questions"):
|
| 103 |
+
q_in = gr.Textbox(label="Ask anything from ingested sources")
|
| 104 |
+
q_btn = gr.Button("Ask")
|
| 105 |
+
q_out = gr.Markdown(label="Answer") # β
Markdown shows multi-line output
|
| 106 |
+
q_btn.click(fn=rag_query, inputs=q_in, outputs=q_out)
|
| 107 |
+
|
| 108 |
+
demo.launch()
|
requirements.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
google-generativeai
|
| 2 |
+
chromadb
|
| 3 |
+
sentence-transformers
|
| 4 |
+
beautifulsoup4
|
| 5 |
+
PyPDF2
|
| 6 |
+
gradio
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
!pip install google-generativeai sentence-transformers chromadb beautifulsoup4 PyPDF2 gradio
|