Update app.py

app.py

@@ -130,7 +130,6 @@
 #             summary = rag_summarize(collected, summarizer)
 #             st.success(summary)
 
-
 import streamlit as st
 import pandas as pd
 import re
@@ -140,36 +139,36 @@ from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.feature_extraction.text import TfidfVectorizer
 from datetime import datetime
 
-# Load and preprocess the dataset
+# ------------------ Load Data ------------------ #
 @st.cache_data
 def load_data():
     df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
-    df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')  # Convert dates
-    df = df.dropna(subset=['status_date'])  # Remove rows with invalid dates
-    df["llama_trend_summary"] = df["llama_trend_summary"].fillna("")  # Clean nulls
+    df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
+    df = df.dropna(subset=['status_date'])
+    df["llama_trend_summary"] = df["llama_trend_summary"].fillna("")
     df["llama_insight"] = df["llama_insight"].fillna("")
-    df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]  # Combine summaries
+    df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
     return df
 
-# Load sentence embedding model + summarization model
+# ------------------ Load Models ------------------ #
 @st.cache_resource
 def load_models():
-    embed_model = SentenceTransformer('all-MiniLM-L6-v2')  # For semantic search
-    summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")  # For final summary
+    embed_model = SentenceTransformer('all-MiniLM-L6-v2')
+    summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
     return embed_model, summarizer
 
-# Generate embeddings from a list of texts
+# ------------------ Compute Embeddings ------------------ #
 @st.cache_data
 def compute_embeddings(texts, _model):
     return _model.encode(texts, show_progress_bar=True)
 
-# Perform semantic search using cosine similarity
-def semantic_search(query, embeddings, model, threshold=0.7):  # Adjusted threshold to 0.7
+# ------------------ Semantic Search ------------------ #
+def semantic_search(query, embeddings, model, threshold=0.5):  # Increased threshold to 0.7
     query_embedding = model.encode([query])
     sims = cosine_similarity(query_embedding, embeddings)[0]
     return [(i, s) for i, s in enumerate(sims) if s > threshold]
 
-# Retrieve top matching texts and summarize them (RAG-like approach)
+# ------------------ RAG Summarizer ------------------ #
 def rag_summarize(texts, summarizer, top_k=5):
     if not texts:
         return "No relevant content to summarize."
@@ -177,13 +176,13 @@ def rag_summarize(texts, summarizer, top_k=5):
     m = vect.fit_transform(texts)
     mean_vec = m.mean(axis=0).A
     scores = cosine_similarity(mean_vec, m).flatten()
-    top_indices = scores.argsort()[::-1][:top_k]  # Pick top-k similar insights
+    top_indices = scores.argsort()[::-1][:top_k]
     ctx = "\n".join(texts[i] for i in top_indices)
     prompt = "summarize: " + ctx[:1024]
     out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
     return out[0]['summary_text']
 
-# Extract month and year from query (e.g., "May 2024")
+# ------------------ Extract Month/Year from Query ------------------ #
 def extract_month_year(q):
     month_map = {m: i for i, m in enumerate(
         ["january", "february", "march", "april", "may", "june",
@@ -194,69 +193,58 @@ def extract_month_year(q):
     yr = int(ym.group()) if ym else None
     return mon, yr
 
-# Try to detect a category mentioned in the query
-def extract_category(q, cats):
-    ql = q.lower()
-    for cat in cats:
-        if pd.isna(cat): continue
-        if any(tok in ql for tok in cat.lower().split()):
-            return cat
-    return None
-
-# ---- Streamlit Interface ---- #
+# ------------------ Topic-Based Matching ------------------ #
+def extract_topic_match(query, df):
+    query_lower = query.lower()
+    matched_rows = df[
+        df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
+        df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
+        df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
+        df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
+    ]
+    return matched_rows
+
+# ------------------ Streamlit UI ------------------ #
 st.set_page_config(page_title="IL Trends Q&A", layout="wide")
 st.title("Illinois Legislative Trends Q&A")
+st.markdown("Ask about **topics** like education, housing, mental health, higher education, etc.\nAlso supports filtering by **month/year**!")
 
-# Load the dataset and models
 df = load_data()
 embed_model, summarizer = load_models()
 
-# User enters question
-query = st.text_input("Ask a question (e.g., ‘trends in higher education in May 2024’):")
+query = st.text_input("🔍 Ask a question (e.g., ‘Higher education in 2024’):")
 
 if query:
-    # Extract date or category from user question
+    # Extract filters
     mon, yr = extract_month_year(query)
-    cats = df['category_&_subcategory_standardized'].unique()
-    cat = extract_category(query, cats)
+    df2 = extract_topic_match(query, df)
 
-    df2 = df.copy()
-
-    # Filter if query includes "opposed"
-    if "opposed" in query.lower():
-        df2 = df2[df2['stance_standardized'].str.lower() == "opposed"]
-        st.info("🔎 Filtering for bills where stance is **opposed**")
-
-    # Filter by detected category
-    if cat:
-        df2 = df2[df2['category_&_subcategory_standardized'] == cat]
-        st.info(f"🔎 Filtering by category: **{cat}**")
+    # Fallback to full dataset if nothing found on topic
+    if df2.empty:
+        df2 = df
 
-    # Filter by year/month if detected
+    # Apply year/month filters
     if yr:
         df2 = df2[df2['status_date'].dt.year == yr]
         if mon:
             df2 = df2[df2['status_date'].dt.month == mon]
-            st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
+            st.info(f"🔎 Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
         else:
-            st.info(f" Filtering by year: **{yr}**")
+            st.info(f"🔎 Filtering by year: **{yr}**")
 
-    # If no data after filtering
     if df2.empty:
         st.warning("No matching records found.")
     else:
-        # Generate semantic matches
         texts = df2['summary_insight'].tolist()
         embs = compute_embeddings(texts, _model=embed_model)
-        res = semantic_search(query, embs, embed_model)  # Uses threshold=0.7
+        res = semantic_search(query, embs, embed_model)
 
         if not res:
             st.warning("No relevant insights found.")
         else:
-            st.subheader("Top Matching Insights")
+            st.subheader(" Top Matching Insights")
             collected = []
 
-            # Display top matches with metadata
             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
                 row = df2.iloc[idx]
                 date = row['status_date'].date()
@@ -266,15 +254,15 @@ if query:
                 stance = row['stance_standardized']
                 trend_summary = row['llama_trend_summary'].strip()
 
-                st.markdown(f"- **Date:** {date} | **Score:** {score:.2f}")
-                st.markdown(f"  - **Category:** {cat_std}")
-                st.markdown(f"  - **Goal:** {goal}")
-                st.markdown(f"  - **Intent:** {intent} | **Stance:** {stance}")
-                st.markdown(f"  > **Trend Summary:** {trend_summary}")
+                st.markdown(f"- ** Date:** {date} | **🔗 Score:** {score:.2f}")
+                st.markdown(f"  - ** Category:** {cat_std}")
+                st.markdown(f"  - ** Goal:** {goal}")
+                st.markdown(f"  - ** Intent:** {intent} | **⚖️ Stance:** {stance}")
+                st.markdown(f"  > ** Trend Summary:** {trend_summary}")
 
                 collected.append(row['summary_insight'])
 
-            # RAG-generated summary from top matching insights
-            st.subheader("RAG-Generated Summary")
+            # RAG Summary
+            st.subheader(" RAG-Generated Summary")
             summary = rag_summarize(collected, summarizer)
             st.success(summary)