Update app.py

app.py

@@ -1,3 +1,135 @@
+# # import streamlit as st
+# # import pandas as pd
+# # import re
+# # from sentence_transformers import SentenceTransformer
+# # from transformers import pipeline
+# # from sklearn.metrics.pairwise import cosine_similarity
+# # from sklearn.feature_extraction.text import TfidfVectorizer
+# # from datetime import datetime
+
+# # # Load data
+# # @st.cache_data
+# # def load_data():
+# #     df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
+# #     df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
+# #     df = df.dropna(subset=['status_date'])
+# #     df["llama_trend_summary"] = df["llama_trend_summary"].fillna("")
+# #     df["llama_insight"] = df["llama_insight"].fillna("")
+# #     df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
+# #     return df
+
+# # # Load models
+# # @st.cache_resource
+# # def load_models():
+# #     embed_model = SentenceTransformer('all-MiniLM-L6-v2')
+# #     summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
+# #     return embed_model, summarizer
+
+# # # Compute embeddings
+# # @st.cache_data
+# # def compute_embeddings(texts, _model):
+# #     return _model.encode(texts, show_progress_bar=True)
+
+# # # Semantic search
+# # def semantic_search(query, embeddings, model, threshold=0.4):
+# #     query_embedding = model.encode([query])
+# #     sims = cosine_similarity(query_embedding, embeddings)[0]
+# #     return [(i, s) for i, s in enumerate(sims) if s > threshold]
+
+# # # RAG summarization
+# # def rag_summarize(texts, summarizer, top_k=5):
+# #     if not texts:
+# #         return "No relevant content to summarize."
+# #     vect = TfidfVectorizer()
+# #     m = vect.fit_transform(texts)
+# #     mean_vec = m.mean(axis=0).A
+# #     scores = cosine_similarity(mean_vec, m).flatten()
+# #     top_indices = scores.argsort()[::-1][:top_k]
+# #     ctx = "\n".join(texts[i] for i in top_indices)
+# #     prompt = "summarize: " + ctx[:1024]
+# #     out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
+# #     return out[0]['summary_text']
+
+# # # Parse month/year
+# # def extract_month_year(q):
+# #     month_map = {m: i for i, m in enumerate(
+# #         ["january", "february", "march", "april", "may", "june",
+# #          "july", "august", "september", "october", "november", "december"], 1)}
+# #     ql = q.lower()
+# #     mon = next((v for k, v in month_map.items() if k in ql), None)
+# #     ym = re.search(r"(19|20)\d{2}", q)
+# #     yr = int(ym.group()) if ym else None
+# #     return mon, yr
+
+# # # Auto-detect category
+# # def extract_category(q, cats):
+# #     ql = q.lower()
+# #     for cat in cats:
+# #         if pd.isna(cat): continue
+# #         if any(tok in ql for tok in cat.lower().split()):
+# #             return cat
+# #     return None
+
+# # # Streamlit UI
+# # st.set_page_config(page_title="IL Trends Q&A", layout="wide")
+# # st.title("Illinois Legislative Trends Q&A")
+
+# # df = load_data()
+# # embed_model, summarizer = load_models()
+
+# # query = st.text_input("Ask a question (e.g., ‘education in May 2024’):")
+
+# # if query:
+# #     mon, yr = extract_month_year(query)
+# #     cats = df['category_&_subcategory_standardized'].unique()
+# #     cat = extract_category(query, cats)
+
+# #     df2 = df.copy()
+# #     if cat:
+# #         df2 = df2[df2['category_&_subcategory_standardized'] == cat]
+# #         st.info(f"Filtering by category: **{cat}**")
+# #     if yr:
+# #         df2 = df2[df2['status_date'].dt.year == yr]
+# #         if mon:
+# #             df2 = df2[df2['status_date'].dt.month == mon]
+# #             st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
+# #         else:
+# #             st.info(f" Filtering by year: **{yr}**")
+
+# #     if df2.empty:
+# #         st.warning("No matching records found.")
+# #     else:
+# #         texts = df2['summary_insight'].tolist()
+# #         embs = compute_embeddings(texts, _model=embed_model)
+# #         res = semantic_search(query, embs, embed_model)
+
+# #         if not res:
+# #             st.warning("No relevant insights found.")
+# #         else:
+# #             st.subheader("Top Matching Insights")
+# #             collected = []
+
+# #             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
+# #                 row = df2.iloc[idx]
+# #                 date = row['status_date'].date()
+# #                 cat_std = row['category_&_subcategory_standardized']
+# #                 goal = row['legislative_goal_standardized']
+# #                 intent = row['intent_standardized']
+# #                 stance = row['stance_standardized']
+# #                 trend_summary = row['llama_trend_summary'].strip()
+
+# #                 st.markdown(f"- **Date:** {date} | **Score:** {score:.2f}")
+# #                 st.markdown(f"  - **Category:** {cat_std}")
+# #                 st.markdown(f"  - **Goal:** {goal}")
+# #                 st.markdown(f"  - **Intent:** {intent} | **Stance:** {stance}")
+# #                 st.markdown(f"  > **Trend Summary:** {trend_summary}")
+
+# #                 collected.append(row['summary_insight'])
+
+# #             st.subheader(" RAG-Generated Summary")
+# #             summary = rag_summarize(collected, summarizer)
+# #             st.success(summary)
+
 # import streamlit as st
 # import pandas as pd
 # import re
@@ -7,7 +139,7 @@
 # from sklearn.feature_extraction.text import TfidfVectorizer
 # from datetime import datetime
 
-# # Load data
+# # loading data 
 # @st.cache_data
 # def load_data():
 #     df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
@@ -18,25 +150,22 @@
 #     df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
 #     return df
 
-# # Load models
 # @st.cache_resource
 # def load_models():
 #     embed_model = SentenceTransformer('all-MiniLM-L6-v2')
 #     summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
 #     return embed_model, summarizer
 
-# # Compute embeddings
 # @st.cache_data
 # def compute_embeddings(texts, _model):
 #     return _model.encode(texts, show_progress_bar=True)
 
-# # Semantic search
-# def semantic_search(query, embeddings, model, threshold=0.4):
+# def semantic_search(query, embeddings, model, threshold=0.5):  
 #     query_embedding = model.encode([query])
 #     sims = cosine_similarity(query_embedding, embeddings)[0]
 #     return [(i, s) for i, s in enumerate(sims) if s > threshold]
 
-# # RAG summarization
+
 # def rag_summarize(texts, summarizer, top_k=5):
 #     if not texts:
 #         return "No relevant content to summarize."
@@ -50,7 +179,6 @@
 #     out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
 #     return out[0]['summary_text']
 
-# # Parse month/year
 # def extract_month_year(q):
 #     month_map = {m: i for i, m in enumerate(
 #         ["january", "february", "march", "april", "may", "june",
@@ -61,40 +189,39 @@
 #     yr = int(ym.group()) if ym else None
 #     return mon, yr
 
-# # Auto-detect category
-# def extract_category(q, cats):
-#     ql = q.lower()
-#     for cat in cats:
-#         if pd.isna(cat): continue
-#         if any(tok in ql for tok in cat.lower().split()):
-#             return cat
-#     return None
+# def extract_topic_match(query, df):
+#     query_lower = query.lower()
+#     matched_rows = df[
+#         df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
+#         df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
+#         df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
+#         df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
+#     ]
+#     return matched_rows
+
 
-# # Streamlit UI
 # st.set_page_config(page_title="IL Trends Q&A", layout="wide")
 # st.title("Illinois Legislative Trends Q&A")
+# st.markdown("Ask about trends in **topics** like education, higher education, etc!")
 
 # df = load_data()
 # embed_model, summarizer = load_models()
 
-# query = st.text_input("Ask a question (e.g., ‘education in May 2024’):")
+# query = st.text_input(" Ask a question (e.g., ‘trends in Higher education in 2024’):")
 
 # if query:
 #     mon, yr = extract_month_year(query)
-#     cats = df['category_&_subcategory_standardized'].unique()
-#     cat = extract_category(query, cats)
+#     df2 = extract_topic_match(query, df)
 
-#     df2 = df.copy()
-#     if cat:
-#         df2 = df2[df2['category_&_subcategory_standardized'] == cat]
-#         st.info(f"Filtering by category: **{cat}**")
+#     if df2.empty:
+#         df2 = df
 #     if yr:
 #         df2 = df2[df2['status_date'].dt.year == yr]
 #         if mon:
 #             df2 = df2[df2['status_date'].dt.month == mon]
 #             st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
 #         else:
-#             st.info(f" Filtering by year: **{yr}**")
+#             st.info(f"Filtering by year: **{yr}**")
 
 #     if df2.empty:
 #         st.warning("No matching records found.")
@@ -106,7 +233,7 @@
 #         if not res:
 #             st.warning("No relevant insights found.")
 #         else:
-#             st.subheader("Top Matching Insights")
+#             st.subheader(" Top Matching Insights")
 #             collected = []
 
 #             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
@@ -118,11 +245,11 @@
 #                 stance = row['stance_standardized']
 #                 trend_summary = row['llama_trend_summary'].strip()
 
-#                 st.markdown(f"- **Date:** {date} | **Score:** {score:.2f}")
-#                 st.markdown(f"  - **Category:** {cat_std}")
-#                 st.markdown(f"  - **Goal:** {goal}")
-#                 st.markdown(f"  - **Intent:** {intent} | **Stance:** {stance}")
-#                 st.markdown(f"  > **Trend Summary:** {trend_summary}")
+#                 st.markdown(f"- ** Date:** {date} | ** Score:** {score:.2f}")
+#                 st.markdown(f"  - ** Category:** {cat_std}")
+#                 st.markdown(f"  - ** Goal:** {goal}")
+#                 st.markdown(f"  - ** Intent:** {intent} | ** Stance:** {stance}")
+#                 st.markdown(f"  > ** Trend Summary:** {trend_summary}")
 
 #                 collected.append(row['summary_insight'])
 
@@ -130,16 +257,19 @@
 #             summary = rag_summarize(collected, summarizer)
 #             st.success(summary)
 
+
 import streamlit as st
 import pandas as pd
 import re
+import dateparser  # for natural language date parsing
 from sentence_transformers import SentenceTransformer
 from transformers import pipeline
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.feature_extraction.text import TfidfVectorizer
 from datetime import datetime
+from io import StringIO
 
-# loading data 
+# Load data
 @st.cache_data
 def load_data():
     df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
@@ -150,22 +280,25 @@ def load_data():
     df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
     return df
 
+# Load models
 @st.cache_resource
 def load_models():
     embed_model = SentenceTransformer('all-MiniLM-L6-v2')
     summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
     return embed_model, summarizer
 
+# Compute embeddings
 @st.cache_data
 def compute_embeddings(texts, _model):
     return _model.encode(texts, show_progress_bar=True)
 
-def semantic_search(query, embeddings, model, threshold=0.5):  
+# Semantic search
+def semantic_search(query, embeddings, model, threshold=0.7):
     query_embedding = model.encode([query])
     sims = cosine_similarity(query_embedding, embeddings)[0]
     return [(i, s) for i, s in enumerate(sims) if s > threshold]
 
-
+# RAG summarization
 def rag_summarize(texts, summarizer, top_k=5):
     if not texts:
         return "No relevant content to summarize."
@@ -179,80 +312,87 @@ def rag_summarize(texts, summarizer, top_k=5):
     out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
     return out[0]['summary_text']
 
-def extract_month_year(q):
+# Enhanced date parsing with dateparser for flexible queries
+def parse_date_from_query(query):
+    dt = dateparser.parse(query, settings={'PREFER_DATES_FROM': 'past'})
+    if dt:
+        return dt.year, dt.month
+    # fallback: regex extract year and month names
     month_map = {m: i for i, m in enumerate(
         ["january", "february", "march", "april", "may", "june",
          "july", "august", "september", "october", "november", "december"], 1)}
-    ql = q.lower()
+    ql = query.lower()
     mon = next((v for k, v in month_map.items() if k in ql), None)
-    ym = re.search(r"(19|20)\d{2}", q)
+    ym = re.search(r"(19|20)\d{2}", query)
     yr = int(ym.group()) if ym else None
-    return mon, yr
+    return yr, mon
 
-def extract_topic_match(query, df):
-    query_lower = query.lower()
-    matched_rows = df[
-        df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
-        df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
-        df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
-        df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
-    ]
-    return matched_rows
+# Simple keyword highlighter
+def highlight_keywords(text, keywords):
+    for kw in keywords:
+        text = re.sub(f"(?i)({re.escape(kw)})", r"**\1**", text)
+    return text
 
+# Streamlit UI
+st.set_page_config(page_title="IL Trends Q&A Enhanced", layout="wide")
+st.title("Illinois Legislative Trends Q&A with Extras")
 
-st.set_page_config(page_title="IL Trends Q&A", layout="wide")
-st.title("Illinois Legislative Trends Q&A")
-st.markdown("Ask about trends in **topics** like education, higher education, etc!")
-
+# Load data & models
 df = load_data()
 embed_model, summarizer = load_models()
 
-query = st.text_input(" Ask a question (e.g., ‘trends in Higher education in 2024’):")
+query = st.text_input("Ask a question (e.g., ‘education in May 2024’, ‘Opposed bills on healthcare’):")
 
 if query:
-    mon, yr = extract_month_year(query)
-    df2 = extract_topic_match(query, df)
-
-    if df2.empty:
-        df2 = df
-    if yr:
-        df2 = df2[df2['status_date'].dt.year == yr]
-        if mon:
-            df2 = df2[df2['status_date'].dt.month == mon]
-            st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
-        else:
-            st.info(f"Filtering by year: **{yr}**")
+    year, month = parse_date_from_query(query)
+
+    # Filter by date if detected
+    df_filtered = df.copy()
+    if year:
+        df_filtered = df_filtered[df_filtered['status_date'].dt.year == year]
+    if month:
+        df_filtered = df_filtered[df_filtered['status_date'].dt.month == month]
 
-    if df2.empty:
-        st.warning("No matching records found.")
+    if df_filtered.empty:
+        st.warning("No data found for the specified time period.")
     else:
-        texts = df2['summary_insight'].tolist()
+        # Compute embeddings for filtered data
+        texts = df_filtered['summary_insight'].tolist()
         embs = compute_embeddings(texts, _model=embed_model)
-        res = semantic_search(query, embs, embed_model)
 
-        if not res:
+        # Perform semantic search with higher threshold
+        results = semantic_search(query, embs, embed_model, threshold=0.7)
+
+        if not results:
             st.warning("No relevant insights found.")
         else:
-            st.subheader(" Top Matching Insights")
-            collected = []
+            st.subheader("Top Matching Insights")
+            collected_texts = []
+            query_keywords = query.lower().split()
 
-            for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
-                row = df2.iloc[idx]
+            for idx, score in sorted(results, key=lambda x: x[1], reverse=True)[:5]:
+                row = df_filtered.iloc[idx]
                 date = row['status_date'].date()
-                cat_std = row['category_&_subcategory_standardized']
+                cat = row['category_&_subcategory_standardized']
                 goal = row['legislative_goal_standardized']
                 intent = row['intent_standardized']
                 stance = row['stance_standardized']
                 trend_summary = row['llama_trend_summary'].strip()
+                summary_text = row['summary_insight']
+
+                highlighted_summary = highlight_keywords(summary_text, query_keywords)
+
+                st.markdown(f"- **Date:** {date} | **Score:** {score:.2f}")
+                st.markdown(f"  - **Category:** {cat}")
+                st.markdown(f"  - **Goal:** {goal}")
+                st.markdown(f"  - **Intent:** {intent} | **Stance:** {stance}")
+                st.markdown(f"  > **Trend Summary:** {trend_summary}")
+                st.markdown(f"  > **Summary Insight:** {highlighted_summary}")
 
-                st.markdown(f"- ** Date:** {date} | ** Score:** {score:.2f}")
-                st.markdown(f"  - ** Category:** {cat_std}")
-                st.markdown(f"  - ** Goal:** {goal}")
-                st.markdown(f"  - ** Intent:** {intent} | ** Stance:** {stance}")
-                st.markdown(f"  > ** Trend Summary:** {trend_summary}")
+                collected_texts.append(summary_text)
 
-                collected.append(row['summary_insight'])
+            # RAG summary of matched results
+            st.subheader("RAG-Generated Summary")
+            rag_summary = rag_summarize(collected_texts, summarizer)
+            st.success(rag_summary)
 
-            st.subheader(" RAG-Generated Summary")
-            summary = rag_summarize(collected, summarizer)
-            st.success(summary)