Update src/streamlit_app.py

src/streamlit_app.py

@@ -1,40 +1,193 @@
-import altair as alt
-import numpy as np
-import pandas as pd
+import os
 import streamlit as st
+import pickle
+import time
+import requests
+from bs4 import BeautifulSoup
+from langchain.llms import HuggingFacePipeline
+from langchain.chains import RetrievalQAWithSourcesChain
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.docstore.document import Document
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+
+from dotenv import load_dotenv
+load_dotenv()
+
+st.title("Research Summarizer and Question Answering Tool 📈")
+st.sidebar.title("Article URLs")
+
+# Add option to choose input method
+input_method = st.sidebar.radio("Input Method:", ["URLs", "Paste Text"])
+
+# Initialize Hugging Face models
+@st.cache_resource
+def initialize_hf_models():
+    # Initialize tokenizer and model for text generation (using FLAN-T5 which is free and good for Q&A)
+    tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-base")
+    model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-base")
+    
+    # Create pipeline
+    pipe = pipeline(
+        "text2text-generation",
+        model=model,
+        tokenizer=tokenizer,
+        max_length=500,
+        temperature=0.9,
+    )
+    
+    # Create LangChain wrapper
+    llm = HuggingFacePipeline(pipeline=pipe)
+    return llm
+
+llm = initialize_hf_models()
+
+# Input fields based on selected method
+if input_method == "URLs":
+    urls = []
+    for i in range(3):
+        url = st.sidebar.text_input(f"URL {i+1}")
+        urls.append(url)
+    process_button_label = "Process URLs"
+else:  # Paste Text
+    st.sidebar.info("💡 Paste article text below (useful when websites block scraping)")
+    pasted_texts = []
+    for i in range(3):
+        text = st.sidebar.text_area(f"Article {i+1} Text:", height=100, key=f"text_{i}")
+        if text.strip():
+            pasted_texts.append(text)
+    process_button_label = "Process Texts"
+
+process_url_clicked = st.sidebar.button(process_button_label)
+file_path = "faiss_store_hf.pkl"
+
+main_placeholder = st.empty()
+
+def extract_text_from_url(url):
+    try:
+        # Enhanced headers to mimic a real browser more closely
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'DNT': '1',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Cache-Control': 'max-age=0',
+        }
+
+        # Add a delay to avoid rate limiting (increased for stricter sites)
+        time.sleep(2)
+
+        response = requests.get(url, headers=headers, timeout=15, allow_redirects=True)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        # Remove unwanted elements
+        for tag in soup(['script', 'style', 'nav', 'header', 'footer', 'ads']):
+            tag.decompose()
+
+        # Extract text from paragraphs
+        paragraphs = soup.find_all('p')
+        text = ' '.join([p.get_text().strip() for p in paragraphs if p.get_text().strip()])
+
+        return text
+    except Exception as e:
+        st.error(f"Error processing {url}: {str(e)}")
+        return None
+
+if process_url_clicked:
+    documents = []
+
+    if input_method == "URLs":
+        # Validate URLs
+        valid_urls = [url for url in urls if url.strip() != ""]
+        if not valid_urls:
+            st.error("Please enter at least one valid URL")
+            st.stop()
+
+        try:
+            # load data from URLs
+            main_placeholder.text("Data Loading...Started...✅✅✅")
+
+            for url in valid_urls:
+                text = extract_text_from_url(url)
+                if text:
+                    doc = Document(page_content=text, metadata={"source": url})
+                    documents.append(doc)
+
+            if not documents:
+                st.error("Could not fetch content from any of the URLs. Please check if the URLs are accessible.")
+                st.info("💡 TIP: If websites are blocking access, try using 'Paste Text' method instead!")
+                st.stop()
+        except Exception as e:
+            st.error(f"An error occurred: {str(e)}")
+            st.stop()
+
+    else:  # Paste Text method
+        if not pasted_texts:
+            st.error("Please paste at least one article text")
+            st.stop()
+
+        try:
+            # load data from pasted text
+            main_placeholder.text("Processing Pasted Text...Started...✅✅✅")
+
+            for idx, text in enumerate(pasted_texts):
+                doc = Document(page_content=text, metadata={"source": f"Pasted Article {idx+1}"})
+                documents.append(doc)
+
+        except Exception as e:
+            st.error(f"An error occurred: {str(e)}")
+            st.stop()
+
+    # Continue with text splitting (same for both methods)
+    if documents:
+        # split data
+        text_splitter = RecursiveCharacterTextSplitter(
+            separators=['\n\n', '\n', '.', ','],
+            chunk_size=1000
+        )
+        main_placeholder.text("Text Splitter...Started...✅✅✅")
+        docs = text_splitter.split_documents(documents)
+
+        if not docs:
+            st.error("No text content could be extracted.")
+            st.stop()
+
+        # create embeddings and save it to FAISS index
+        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+        vectorstore = FAISS.from_documents(docs, embeddings)
+        main_placeholder.text("Embedding Vector Started Building...✅✅✅")
+        time.sleep(2)
+
+        # Save the FAISS index to a pickle file
+        with open(file_path, "wb") as f:
+            pickle.dump(vectorstore, f)
+
+query = main_placeholder.text_input("Question: ")
+if query:
+    if os.path.exists(file_path):
+        with open(file_path, "rb") as f:
+            vectorstore = pickle.load(f)
+            chain = RetrievalQAWithSourcesChain.from_llm(llm=llm, retriever=vectorstore.as_retriever())
+            result = chain({"question": query}, return_only_outputs=True)
+            # result will be a dictionary of this format --> {"answer": "", "sources": [] }
+            st.header("Answer")
+            st.write(result["answer"])
+
+            # Display sources, if available
+            sources = result.get("sources", "")
+            if sources:
+                st.subheader("Sources:")
+                sources_list = sources.split("\n")  # Split the sources by newline
+                for source in sources_list:
+                    st.write(source)
+
+
 
-"""
-# Welcome to Streamlit!
-
-Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
-
-In the meantime, below is an example of what you can do with just a few lines of code:
-"""
-
-num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
-num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
-
-indices = np.linspace(0, 1, num_points)
-theta = 2 * np.pi * num_turns * indices
-radius = indices
-
-x = radius * np.cos(theta)
-y = radius * np.sin(theta)
-
-df = pd.DataFrame({
-    "x": x,
-    "y": y,
-    "idx": indices,
-    "rand": np.random.randn(num_points),
-})
-
-st.altair_chart(alt.Chart(df, height=700, width=700)
-    .mark_point(filled=True)
-    .encode(
-        x=alt.X("x", axis=None),
-        y=alt.Y("y", axis=None),
-        color=alt.Color("idx", legend=None, scale=alt.Scale()),
-        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
-    ))