Update app.py

app.py

@@ -1,148 +1,3 @@
-# import streamlit as st
-# import pandas as pd
-# import re
-# from sentence_transformers import SentenceTransformer
-# from transformers import pipeline
-# from sklearn.metrics.pairwise import cosine_similarity
-# from sklearn.feature_extraction.text import TfidfVectorizer
-# from datetime import datetime
-
-# def clean_text(text):
-#     text = re.sub(r"(?i)(here is|here are) the requested output[s]*[:]*", "", text)
-#     text = re.sub(r"(?i)let me know if you'd like.*", "", text)
-#     text = re.sub(r"(?i)trend summary[:]*", "", text)
-#     text = re.sub(r"(?i)actionable insight[:]*", "", text)
-#     return text.strip()
-
-# @st.cache_data
-# def load_data():
-#     df = pd.read_csv("Illinois_Entire_Data_Insights_Final_v2.csv")
-#     df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
-#     df = df.dropna(subset=['status_date'])
-#     df["llama_trend_summary"] = df["llama_trend_summary"].fillna("")
-#     df["llama_insight"] = df["llama_insight"].fillna("")
-#     df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
-#     return df
-
-# @st.cache_resource
-# def load_models():
-#     embed_model = SentenceTransformer('all-MiniLM-L6-v2')
-#     summarizer = pipeline("summarization", model="t5-small", tokenizer="t5-small")
-#     return embed_model, summarizer
-
-# @st.cache_data
-# def compute_embeddings(texts, _model):
-#     return _model.encode(texts, show_progress_bar=True)
-
-# def semantic_search(query, embeddings, model, threshold=0.5):  
-#     query_embedding = model.encode([query])
-#     sims = cosine_similarity(query_embedding, embeddings)[0]
-#     return [(i, s) for i, s in enumerate(sims) if s > threshold]
-
-# def rag_summarize(texts, summarizer, top_k=5):
-#     if not texts:
-#         return "No relevant content to summarize."
-#     vect = TfidfVectorizer()
-#     m = vect.fit_transform(texts)
-#     mean_vec = m.mean(axis=0).A
-#     scores = cosine_similarity(mean_vec, m).flatten()
-#     top_indices = scores.argsort()[::-1][:top_k]
-#     ctx = "\n".join(texts[i] for i in top_indices)
-#     prompt = "summarize: " + ctx[:1024]
-#     out = summarizer(prompt, max_length=60, min_length=30, do_sample=False)
-#     return out[0]['summary_text']
-
-# def extract_month_year(q):
-#     month_map = {m: i for i, m in enumerate(
-#         ["january", "february", "march", "april", "may", "june",
-#          "july", "august", "september", "october", "november", "december"], 1)}
-#     ql = q.lower()
-#     mon = next((v for k, v in month_map.items() if k in ql), None)
-#     ym = re.search(r"(19|20)\d{2}", q)
-#     yr = int(ym.group()) if ym else None
-#     return mon, yr
-
-# def extract_topic_match(query, df):
-#     query_lower = query.lower()
-#     return df[
-#         df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
-#         df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
-#         df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
-#         df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
-#     ]
-
-# st.set_page_config(page_title="IL Legislative Trends Q&A", layout="wide")
-# st.title("Illinois Legislative Trends Q&A")
-# st.markdown("Ask about trends in topics like higher education, funding, etc.")
-
-# df = load_data()
-# embed_model, summarizer = load_models()
-
-# query = st.text_input("Ask a question (e.g., ‘Trends in higher education in 2024’):")
-
-# if query:
-#     mon, yr = extract_month_year(query)
-#     df2 = extract_topic_match(query, df)
-
-#     if df2.empty:
-#         df2 = df
-#     if yr:
-#         df2 = df2[df2['status_date'].dt.year == yr]
-#         if mon:
-#             df2 = df2[df2['status_date'].dt.month == mon]
-#             st.info(f" Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
-#         else:
-#             st.info(f" Filtering by year: **{yr}**")
-
-#     if df2.empty:
-#         st.warning("No matching records found.")
-#     else:
-#         texts = df2['summary_insight'].tolist()
-#         embs = compute_embeddings(texts, _model=embed_model)
-#         res = semantic_search(query, embs, embed_model, threshold=0.5)
-
-#         if not res:
-#             st.warning("No relevant insights found.")
-#         else:
-#             st.subheader(" Top Matching Insights")
-#             collected = []
-
-#             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
-#                 row = df2.iloc[idx]
-#                 date = row['status_date'].date()
-#                 bill_number = row['bill_number']
-#                 full_url = row['url']
-#                 cat_std = row['Category & Subcategory']
-#                 goal = row['Legislative Goal']
-#                 impact = row['Policy Impact Areas']
-#                 provision = row['Key Provisions']
-#                 intent = row['Intent']
-#                 stance = row['Stance']
-#                 description = row['description']
-#                 summary = row['summary']
-
-#                 trend = clean_text(row['llama_trend_summary'])
-#                 insight = clean_text(row['llama_insight'])
-
-#                 st.markdown(f"**Date:** {date} | **Bill Number:** {bill_number} | **Score:** {score:.2f}")
-#                 st.markdown(f"**Category:** {cat_std}")
-#                 st.markdown(f"**Goal:** {goal}")
-#                 st.markdown(f"**Intent:** {intent} | **Stance:** {stance}")
-#                 st.markdown(f"**Policy Impacy Area:** {impact}")
-#                 st.markdown(f"**Key Provision:** {provision}")
-#                 st.markdown(f"**Description:** {description}")
-#                 st.markdown(f"**Summary:** {summary}")
-#                 st.markdown(f"Trend Summary:{trend}")
-#                 st.markdown(f"Actionable Insight:{insight}")
-#                 st.markdown(f"[View Full Bill Text]({full_url})\n")
-#                 st.divider()
-
-#                 collected.append(row['summary_insight'])
-
-#             st.subheader("RAG-Generated Overall Summary")
-#             summary = rag_summarize(collected, summarizer)
-#             st.success(summary)
-
 import streamlit as st
 import pandas as pd
 import re
@@ -166,11 +21,7 @@ def load_data():
     df = df.dropna(subset=['status_date'])
     df["llama_trend_summary"] = df["llama_trend_summary"].fillna("")
     df["llama_insight"] = df["llama_insight"].fillna("")
-    df["description"] = df["description"].fillna("")
-    df["summary"] = df["summary"].fillna("")
-    df["Policy Impact Areas"] = df["Policy Impact Areas"].fillna("")
-    df["Key Provisions"] = df["Key Provisions"].fillna("")
-    # Optional: Add more preprocessing if needed
+    df["summary_insight"] = df["llama_trend_summary"] + "\n" + df["llama_insight"]
     return df
 
 @st.cache_resource
@@ -215,7 +66,6 @@ def extract_topic_match(query, df):
     query_lower = query.lower()
     return df[
         df['category_&_subcategory_standardized'].fillna('').str.lower().str.contains(query_lower) |
-        df['category_granular_standardized'].fillna('').str.lower().str.contains(query_lower) |
         df['intent_standardized'].fillna('').str.lower().str.contains(query_lower) |
         df['legislative_goal_standardized'].fillna('').str.lower().str.contains(query_lower) |
         df['policy_impact_areas_standardized'].fillna('').str.lower().str.contains(query_lower)
@@ -240,29 +90,21 @@ if query:
         df2 = df2[df2['status_date'].dt.year == yr]
         if mon:
             df2 = df2[df2['status_date'].dt.month == mon]
-            st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
+            st.info(f" Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
         else:
-            st.info(f"Filtering by year: **{yr}**")
+            st.info(f" Filtering by year: **{yr}**")
 
     if df2.empty:
         st.warning("No matching records found.")
     else:
-        texts = df2.apply(lambda row: "\n".join([
-            clean_text(str(row.get('llama_trend_summary', ''))),
-            clean_text(str(row.get('llama_insight', ''))),
-            str(row.get('summary', '')),
-            str(row.get('description', '')),
-            str(row.get('Policy Impact Areas', '')),
-            str(row.get('Key Provisions', ''))
-        ]), axis=1).tolist()
-
+        texts = df2['summary_insight'].tolist()
         embs = compute_embeddings(texts, _model=embed_model)
         res = semantic_search(query, embs, embed_model, threshold=0.5)
 
         if not res:
             st.warning("No relevant insights found.")
         else:
-            st.subheader("Top Matching Insights")
+            st.subheader(" Top Matching Insights")
             collected = []
 
             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:5]:
@@ -278,6 +120,7 @@ if query:
                 stance = row['Stance']
                 description = row['description']
                 summary = row['summary']
+
                 trend = clean_text(row['llama_trend_summary'])
                 insight = clean_text(row['llama_insight'])
 
@@ -285,24 +128,19 @@ if query:
                 st.markdown(f"**Category:** {cat_std}")
                 st.markdown(f"**Goal:** {goal}")
                 st.markdown(f"**Intent:** {intent} | **Stance:** {stance}")
-                st.markdown(f"**Policy Impact Area:** {impact}")
+                st.markdown(f"**Policy Impacy Area:** {impact}")
                 st.markdown(f"**Key Provision:** {provision}")
                 st.markdown(f"**Description:** {description}")
                 st.markdown(f"**Summary:** {summary}")
-                st.markdown(f"Trend Summary: {trend}")
-                st.markdown(f"Actionable Insight: {insight}")
-                st.markdown(f"[View Full Bill Text]({full_url})")
+                st.markdown(f"Trend Summary:{trend}")
+                st.markdown(f"Actionable Insight:{insight}")
+                st.markdown(f"[View Full Bill Text]({full_url})\n")
                 st.divider()
 
-                collected.append("\n".join([
-                    trend, insight, summary, description, impact, provision
-                ]))
-
-            with st.expander("View summarized insights context"):
-                for i, t in enumerate(collected[:5]):
-                    st.markdown(f"**Insight #{i+1}:**\n{t}\n")
+                collected.append(row['summary_insight'])
 
             st.subheader("RAG-Generated Overall Summary")
             summary = rag_summarize(collected, summarizer)
             st.success(summary)
 
+