Update app.py

app.py

@@ -1,84 +1,99 @@
-import streamlit as st
-from selenium import webdriver
-from selenium.webdriver.chrome.service import Service
-from webdriver_manager.chrome import ChromeDriverManager
-from selenium.webdriver.chrome.options import Options
-import time
-
-from sentence_transformers import SentenceTransformer
-import faiss
-import numpy as np
-from transformers import pipeline
-
-# -------------------------------
-# 1. Setup Selenium (Headless Chrome for Hugging Face/Streamlit)
-# -------------------------------
-def init_driver():
-    chrome_options = Options()
-    chrome_options.add_argument("--headless")  
-    chrome_options.add_argument("--disable-gpu")
-    chrome_options.add_argument("--no-sandbox")
-    chrome_options.add_argument("--disable-dev-shm-usage")
-
-    service = Service(ChromeDriverManager().install())
-    driver = webdriver.Chrome(service=service, options=chrome_options)
-    return driver
-
-# -------------------------------
-# 2. Scrape website text with Selenium
-# -------------------------------
-def scrape_website(url):
-    driver = init_driver()
-    driver.get(url)
-    time.sleep(3)  # wait for JS to load
-    text = driver.page_source  # raw HTML
-    driver.quit()
-    return text
-
-# -------------------------------
-# 3. Embed and store in FAISS
-# -------------------------------
-embedder = SentenceTransformer("all-MiniLM-L6-v2")
-dimension = 384
-index = faiss.IndexFlatL2(dimension)
-
-documents = []
-
-def add_to_faiss(text):
-    global documents
-    embedding = embedder.encode([text])
-    index.add(np.array(embedding, dtype="float32"))
-    documents.append(text)
-
-def retrieve(query, k=1):
-    q_emb = embedder.encode([query])
-    D, I = index.search(np.array(q_emb, dtype="float32"), k)
-    return [documents[i] for i in I[0]]
-
-# -------------------------------
-# 4. QA Model (FLAN-T5-small)
-# -------------------------------
-qa_pipeline = pipeline("text2text-generation", model="google/flan-t5-small")
-
-def answer_query(query):
-    context_docs = retrieve(query, k=1)
-    context = " ".join(context_docs)
-    prompt = f"Answer the question based on context:\nContext: {context}\nQuestion: {query}"
-    result = qa_pipeline(prompt, max_length=256, do_sample=False)
-    return result[0]['generated_text']
-
-# -------------------------------
-# 5. Streamlit App
-# -------------------------------
-st.title("🌐 Web Scraping + RAG (Selenium + FLAN-T5-small)")
-
-url = st.text_input("Enter website URL:")
-if url and st.button("Scrape & Index"):
-    scraped_text = scrape_website(url)
-    add_to_faiss(scraped_text)
-    st.success("✅ Website scraped and indexed successfully!")
-
-query = st.text_input("Ask a question:")
-if query and st.button("Get Answer"):
-    answer = answer_query(query)
-    st.write("**Answer:**", answer)
+import streamlit as st
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+import time
+
+# -------------------------
+# FIX for huggingface_hub cached_download issue
+# -------------------------
+import huggingface_hub
+if not hasattr(huggingface_hub, "cached_download"):
+    from huggingface_hub import hf_hub_download
+    huggingface_hub.cached_download = hf_hub_download
+
+# -------------------------
+# RAG + NLP libraries
+# -------------------------
+from sentence_transformers import SentenceTransformer
+import faiss
+import numpy as np
+from transformers import pipeline
+
+
+# -------------------------
+# 1️⃣ Function: Scrape website using Selenium
+# -------------------------
+def scrape_with_selenium(url: str):
+    chrome_options = Options()
+    chrome_options.add_argument("--headless")
+    chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_argument("--disable-dev-shm-usage")
+
+    driver = webdriver.Chrome(options=chrome_options)
+    driver.get(url)
+    time.sleep(2)
+
+    # Scrape all visible text
+    paragraphs = driver.find_elements("tag name", "p")
+    text_data = [p.text for p in paragraphs if p.text.strip()]
+    driver.quit()
+
+    return text_data
+
+
+# -------------------------
+# 2️⃣ Function: Build FAISS Index
+# -------------------------
+def build_faiss_index(text_data):
+    model = SentenceTransformer("all-MiniLM-L6-v2")
+    embeddings = model.encode(text_data, convert_to_numpy=True)
+
+    dim = embeddings.shape[1]
+    index = faiss.IndexFlatL2(dim)
+    index.add(embeddings)
+
+    return model, index, text_data
+
+
+# -------------------------
+# 3️⃣ Function: Query RAG
+# -------------------------
+def query_rag(question, model, index, text_data):
+    q_embedding = model.encode([question], convert_to_numpy=True)
+    D, I = index.search(q_embedding, k=3)
+    retrieved = [text_data[i] for i in I[0]]
+
+    # Generate answer using Flan-T5
+    generator = pipeline("text2text-generation", model="google/flan-t5-small")
+    context = " ".join(retrieved)
+    prompt = f"Answer the question using the context:\nContext: {context}\nQuestion: {question}"
+    answer = generator(prompt, max_length=150, do_sample=True)[0]["generated_text"]
+
+    return answer, retrieved
+
+
+# -------------------------
+# 4️⃣ Streamlit UI
+# -------------------------
+st.title("🚀 Web Scraping + RAG with Selenium")
+
+url = st.text_input("Enter a website URL:", "https://quotes.toscrape.com/")
+if st.button("Scrape Website"):
+    with st.spinner("Scraping website..."):
+        scraped_text = scrape_with_selenium(url)
+        st.success(f"✅ Scraped {len(scraped_text)} paragraphs!")
+
+        st.session_state["scraped_text"] = scraped_text
+
+if "scraped_text" in st.session_state:
+    question = st.text_input("Ask a question based on scraped data:")
+    if st.button("Get Answer"):
+        model, index, text_data = build_faiss_index(st.session_state["scraped_text"])
+        answer, retrieved = query_rag(question, model, index, text_data)
+
+        st.subheader("🔍 Retrieved Context")
+        for r in retrieved:
+            st.write("-", r)
+
+        st.subheader("💡 Answer")
+        st.write(answer)