Update app.py

app.py

@@ -675,367 +675,195 @@
 
 #BART
 
-# import streamlit as st
-# import pandas as pd
-# import re
-# from sentence_transformers import SentenceTransformer
-# from transformers import pipeline
-# from sklearn.metrics.pairwise import cosine_similarity
-# from sklearn.feature_extraction.text import TfidfVectorizer
-# from datetime import datetime
-
-# def clean_text(text):
-#     text = re.sub(r"(?i)(here is|here are) the requested output[s]*[:]*", "", text)
-#     text = re.sub(r"(?i)let me know if you'd like.*", "", text)
-#     text = re.sub(r"(?i)trend summary[:]*", "", text)
-#     text = re.sub(r"(?i)actionable insight[:]*", "", text)
-#     return text.strip()
-
-# @st.cache_data
-# def load_data():
-#     df = pd.read_csv("Illinois_Education_Bills_Summarized_With Features_2021_2025_07182025.csv")
-#     df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
-#     df = df.dropna(subset=['status_date'])
-
-#     for col in ["Legislative Goal", "Policy Impact Areas", "Key Provisions",
-#                 "Intended Beneficiaries", "Potential Impact", "description"]:
-#         df[col] = df[col].fillna("")
-
-#     df["combined_text"] = (
-#         "Legislative Goal: " + df["Legislative Goal"] + "\n" +
-#         "Policy Impact Areas: " + df["Policy Impact Areas"] + "\n" +
-#         "Key Provisions: " + df["Key Provisions"] + "\n" +
-#         "Intended Beneficiaries: " + df["Intended Beneficiaries"] + "\n" +
-#         "Potential Impact: " + df["Potential Impact"] + "\n" +
-#         "Description: " + df["description"]
-#     )
-
-#     return df
-
-# @st.cache_resource
-# def load_models():
-#     embed_model = SentenceTransformer('all-MiniLM-L6-v2')
-#     # Changed summarization model to facebook/bart-large-cnn for better summary quality
-#     summarizer = pipeline("summarization", model="facebook/bart-large-cnn", tokenizer="facebook/bart-large-cnn")
-#     return embed_model, summarizer
-
-# @st.cache_data
-# def compute_embeddings(texts, _model):
-#     return _model.encode(texts, show_progress_bar=True)
-
-# def semantic_search(query, embeddings, model, threshold=0.5):  
-#     query_embedding = model.encode([query])
-#     sims = cosine_similarity(query_embedding, embeddings)[0]
-#     return [(i, s) for i, s in enumerate(sims) if s > threshold]
-
-# def rag_summarize(texts, summarizer, top_k=5):
-#     if not texts:
-#         return "No relevant content to summarize."
-#     vect = TfidfVectorizer()
-#     m = vect.fit_transform(texts)
-#     mean_vec = m.mean(axis=0).A
-#     scores = cosine_similarity(mean_vec, m).flatten()
-#     top_indices = scores.argsort()[::-1][:top_k]
-#     ctx = "\n".join(texts[i] for i in top_indices)
-#     prompt = "summarize: " + ctx[:1024]
-#     out = summarizer(prompt, max_length=200, min_length=80, do_sample=False)  
-#     return out[0]['summary_text']
-
-# def extract_month_year(q):
-#     month_map = {m: i for i, m in enumerate(
-#         ["january", "february", "march", "april", "may", "june",
-#          "july", "august", "september", "october", "november", "december"], 1)}
-#     ql = q.lower()
-#     mon = next((v for k, v in month_map.items() if k in ql), None)
-#     ym = re.search(r"(19|20)\d{2}", q)
-#     yr = int(ym.group()) if ym else None
-#     return mon, yr
-
-# def extract_date_range(query):
-#     month_map = {
-#         "january": 1, "february": 2, "march": 3, "april": 4, "may": 5, "june": 6,
-#         "july": 7, "august": 8, "september": 9, "october": 10, "november": 11, "december": 12
-#     }
-
-#     patterns = [
-#         r"(?i)(?:from|between)?\s*([a-zA-Z]+)\s+(\d{4})\s*(?:to|through|and|-)\s*([a-zA-Z]+)\s+(\d{4})",
-#     ]
-
-#     for pattern in patterns:
-#         match = re.search(pattern, query)
-#         if match:
-#             start_month_str, start_year = match.group(1).lower(), int(match.group(2))
-#             end_month_str, end_year = match.group(3).lower(), int(match.group(4))
-
-#             start_month = month_map.get(start_month_str)
-#             end_month = month_map.get(end_month_str)
-
-#             if start_month and end_month:
-#                 start_date = datetime(start_year, start_month, 1)
-#                 end_date = datetime(end_year, end_month, 28)
-#                 return start_date, end_date
-
-#     return None, None
-
-
-# def extract_topic_match(query, df):
-#     query_lower = query.lower()
-#     return df[
-#         df['Category & Subcategory'].fillna('').str.lower().str.contains(query_lower) |
-#         df['Intent'].fillna('').str.lower().str.contains(query_lower) |
-#         df['Legislative Goal'].fillna('').str.lower().str.contains(query_lower) |
-#         df['Policy Impact Areas'].fillna('').str.lower().str.contains(query_lower) |
-#         df['Key Provisions'].fillna('').str.lower().str.contains(query_lower) |
-#         df['Potential Impact'].fillna('').str.lower().str.contains(query_lower)
-#     ]
-
-
-# st.set_page_config(page_title="IL Legislative Trends Q&A", layout="wide")
-# st.title("Illinois Legislative Trends Q&A")
-# st.markdown("Ask about trends in topics like higher education, funding, etc.")
-
-# df = load_data()
-# embed_model, summarizer = load_models()
-
-# query = st.text_input("Ask a question (e.g., ‘Trends from Jan 2024 to May 2025’):")
-
-# if query:
-#     start_date, end_date = extract_date_range(query)
-#     df2 = extract_topic_match(query, df)
-
-#     if df2.empty:
-#         df2 = df
-
-#     if start_date and end_date:
-#         df2 = df2[(df2['status_date'] >= start_date) & (df2['status_date'] <= end_date)]
-#         st.info(f"Filtering between: **{start_date:%B %Y}** and **{end_date:%B %Y}**")
-#     else:
-#         mon, yr = extract_month_year(query)
-#         if yr:
-#             df2 = df2[df2['status_date'].dt.year == yr]
-#             if mon:
-#                 df2 = df2[df2['status_date'].dt.month == mon]
-#                 st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
-#             else:
-#                 st.info(f"Filtering by year: **{yr}**")
-
-#     if df2.empty:
-#         st.warning("No matching records found.")
-#     else:
-#         texts = df2['combined_text'].tolist()
-#         embs = compute_embeddings(texts, _model=embed_model)
-#         res = semantic_search(query, embs, embed_model, threshold=0.5)
-
-#         if not res:
-#             st.warning("No relevant insights found.")
-#         else:
-#             st.subheader("Top Matching Insights")
-#             collected = []
-
-#             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:10]: 
-#                 row = df2.iloc[idx]
-#                 date = row['status_date'].date()
-#                 bill_number = row['bill_number']
-#                 full_url = row['url']
-#                 cat = row.get('Category & Subcategory', '')
-#                 bene = row.get('Intended Beneficiaries', '')
-#                 goal = row.get('Legislative Goal', '')
-#                 impact = row.get('Policy Impact Areas', '')
-#                 provision = row.get('Key Provisions', '')
-#                 intent = row.get('Intent', '')
-#                 stance = row.get('Stance', '')
-#                 description = row.get('description', '')
-
-#                 st.markdown(f"**Date:** {date} | **Bill Number:** {bill_number} | **Score:** {score:.2f}")
-#                 st.markdown(f"**Category:** {cat}")
-#                 st.markdown(f"**Intended Beneficiaries:** {bene}")
-#                 st.markdown(f"**Goal:** {goal}")
-#                 st.markdown(f"**Intent:** {intent} | **Stance:** {stance}")
-#                 st.markdown(f"**Policy Impact Area:** {impact}")
-#                 st.markdown(f"**Key Provision:** {provision}")
-#                 st.markdown(f"**Description:** {description}")
-#                 st.markdown(f"[View Full Bill Text]({full_url})\n")
-#                 st.divider()
-
-#                 collected.append(row['combined_text'])
-
-#             st.subheader("RAG-Generated Overall Summary")
-#             summary = rag_summarize(collected, summarizer)
-#             st.success(summary)
-
+ import streamlit as st
+ import pandas as pd
+ import re
+ from sentence_transformers import SentenceTransformer
+ from transformers import pipeline
+ from sklearn.metrics.pairwise import cosine_similarity
+ from sklearn.feature_extraction.text import TfidfVectorizer
+ from datetime import datetime
+
+ def clean_text(text):
+     text = re.sub(r"(?i)(here is|here are) the requested output[s]*[:]*", "", text)
+     text = re.sub(r"(?i)let me know if you'd like.*", "", text)
+     text = re.sub(r"(?i)trend summary[:]*", "", text)
+     text = re.sub(r"(?i)actionable insight[:]*", "", text)
+     return text.strip()
+
+ @st.cache_data
+ def load_data():
+     df = pd.read_csv("Illinois_Education_Bills_Summarized_With Features_2021_2025_07182025.csv")
+     df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
+     df = df.dropna(subset=['status_date'])
+
+     for col in ["Legislative Goal", "Policy Impact Areas", "Key Provisions",
+                 "Intended Beneficiaries", "Potential Impact", "description"]:
+         df[col] = df[col].fillna("")
+
+     df["combined_text"] = (
+         "Legislative Goal: " + df["Legislative Goal"] + "\n" +
+         "Policy Impact Areas: " + df["Policy Impact Areas"] + "\n" +
+         "Key Provisions: " + df["Key Provisions"] + "\n" +
+         "Intended Beneficiaries: " + df["Intended Beneficiaries"] + "\n" +
+         "Potential Impact: " + df["Potential Impact"] + "\n" +
+         "Description: " + df["description"]
+     )
+
+     return df
+
+ @st.cache_resource
+ def load_models():
+     embed_model = SentenceTransformer('all-MiniLM-L6-v2')
+     # Changed summarization model to facebook/bart-large-cnn for better summary quality
+     summarizer = pipeline("summarization", model="facebook/bart-large-cnn", tokenizer="facebook/bart-large-cnn")
+     return embed_model, summarizer
+
+ @st.cache_data
+ def compute_embeddings(texts, _model):
+     return _model.encode(texts, show_progress_bar=True)
+
+ def semantic_search(query, embeddings, model, threshold=0.5):  
+     query_embedding = model.encode([query])
+     sims = cosine_similarity(query_embedding, embeddings)[0]
+     return [(i, s) for i, s in enumerate(sims) if s > threshold]
+
+ def rag_summarize(texts, summarizer, top_k=5):
+     if not texts:
+         return "No relevant content to summarize."
+     vect = TfidfVectorizer()
+     m = vect.fit_transform(texts)
+     mean_vec = m.mean(axis=0).A
+     scores = cosine_similarity(mean_vec, m).flatten()
+     top_indices = scores.argsort()[::-1][:top_k]
+     ctx = "\n".join(texts[i] for i in top_indices)
+     prompt = "summarize: " + ctx[:1024]
+     out = summarizer(prompt, max_length=200, min_length=80, do_sample=False)  
+     return out[0]['summary_text']
+
+ def extract_month_year(q):
+     month_map = {m: i for i, m in enumerate(
+         ["january", "february", "march", "april", "may", "june",
+          "july", "august", "september", "october", "november", "december"], 1)}
+     ql = q.lower()
+     mon = next((v for k, v in month_map.items() if k in ql), None)
+     ym = re.search(r"(19|20)\d{2}", q)
+     yr = int(ym.group()) if ym else None
+     return mon, yr
+
+ def extract_date_range(query):
+     month_map = {
+         "january": 1, "february": 2, "march": 3, "april": 4, "may": 5, "june": 6,
+         "july": 7, "august": 8, "september": 9, "october": 10, "november": 11, "december": 12
+     }
+
+     patterns = [
+         r"(?i)(?:from|between)?\s*([a-zA-Z]+)\s+(\d{4})\s*(?:to|through|and|-)\s*([a-zA-Z]+)\s+(\d{4})",
+     ]
+
+     for pattern in patterns:
+         match = re.search(pattern, query)
+         if match:
+             start_month_str, start_year = match.group(1).lower(), int(match.group(2))
+             end_month_str, end_year = match.group(3).lower(), int(match.group(4))
+
+             start_month = month_map.get(start_month_str)
+             end_month = month_map.get(end_month_str)
+
+             if start_month and end_month:
+                 start_date = datetime(start_year, start_month, 1)
+                 end_date = datetime(end_year, end_month, 28)
+                 return start_date, end_date
+
+     return None, None
+
+
+ def extract_topic_match(query, df):
+     query_lower = query.lower()
+     return df[
+         df['Category & Subcategory'].fillna('').str.lower().str.contains(query_lower) |
+         df['Intent'].fillna('').str.lower().str.contains(query_lower) |
+         df['Legislative Goal'].fillna('').str.lower().str.contains(query_lower) |
+         df['Policy Impact Areas'].fillna('').str.lower().str.contains(query_lower) |
+         df['Key Provisions'].fillna('').str.lower().str.contains(query_lower) |
+         df['Potential Impact'].fillna('').str.lower().str.contains(query_lower)
+     ]
+
+
+ st.set_page_config(page_title="IL Legislative Trends Q&A", layout="wide")
+ st.title("Illinois Legislative Trends Q&A")
+ st.markdown("Ask about trends in topics like higher education, funding, etc.")
+
+ df = load_data()
+ embed_model, summarizer = load_models()
+
+ query = st.text_input("Ask a question (e.g., ‘Trends from Jan 2024 to May 2025’):")
+
+ if query:
+     start_date, end_date = extract_date_range(query)
+     df2 = extract_topic_match(query, df)
+
+     if df2.empty:
+         df2 = df
+
+     if start_date and end_date:
+         df2 = df2[(df2['status_date'] >= start_date) & (df2['status_date'] <= end_date)]
+         st.info(f"Filtering between: **{start_date:%B %Y}** and **{end_date:%B %Y}**")
+     else:
+         mon, yr = extract_month_year(query)
+         if yr:
+             df2 = df2[df2['status_date'].dt.year == yr]
+             if mon:
+                 df2 = df2[df2['status_date'].dt.month == mon]
+                 st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
+             else:
+                 st.info(f"Filtering by year: **{yr}**")
+
+     if df2.empty:
+         st.warning("No matching records found.")
+     else:
+         texts = df2['combined_text'].tolist()
+         embs = compute_embeddings(texts, _model=embed_model)
+         res = semantic_search(query, embs, embed_model, threshold=0.5)
+
+         if not res:
+             st.warning("No relevant insights found.")
+         else:
+             st.subheader("Top Matching Insights")
+             collected = []
+
+             for idx, score in sorted(res, key=lambda x: x[1], reverse=True)[:10]: 
+                 row = df2.iloc[idx]
+                 date = row['status_date'].date()
+                 bill_number = row['bill_number']
+                 full_url = row['url']
+                 cat = row.get('Category & Subcategory', '')
+                 bene = row.get('Intended Beneficiaries', '')
+                 goal = row.get('Legislative Goal', '')
+                 impact = row.get('Policy Impact Areas', '')
+                 provision = row.get('Key Provisions', '')
+                 intent = row.get('Intent', '')
+                 stance = row.get('Stance', '')
+                 description = row.get('description', '')
+
+                 st.markdown(f"**Date:** {date} | **Bill Number:** {bill_number} | **Score:** {score:.2f}")
+                 st.markdown(f"**Category:** {cat}")
+                 st.markdown(f"**Intended Beneficiaries:** {bene}")
+                 st.markdown(f"**Goal:** {goal}")
+                 st.markdown(f"**Intent:** {intent} | **Stance:** {stance}")
+                 st.markdown(f"**Policy Impact Area:** {impact}")
+                 st.markdown(f"**Key Provision:** {provision}")
+                 st.markdown(f"**Description:** {description}")
+                 st.markdown(f"[View Full Bill Text]({full_url})\n")
+                 st.divider()
+
+                 collected.append(row['combined_text'])
+
+             st.subheader("RAG-Generated Overall Summary")
+             summary = rag_summarize(collected, summarizer)
+             st.success(summary)
 
 
-#bartv2 - trying to make summary better
-
-import streamlit as st
-import pandas as pd
-import re
-from sentence_transformers import SentenceTransformer
-from transformers import pipeline
-from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.feature_extraction.text import TfidfVectorizer
-from datetime import datetime
-
-def clean_text(text):
-    text = re.sub(r"(?i)(here is|here are) the requested output[s]*[:]*", "", text)
-    text = re.sub(r"(?i)let me know if you'd like.*", "", text)
-    text = re.sub(r"(?i)trend summary[:]*", "", text)
-    text = re.sub(r"(?i)actionable insight[:]*", "", text)
-    return text.strip()
-
-@st.cache_data
-def load_data():
-    df = pd.read_csv("Illinois_Education_Bills_Summarized_With Features_2021_2025_07182025.csv")
-    df['status_date'] = pd.to_datetime(df['status_date'], format='%d-%m-%Y', errors='coerce')
-    df = df.dropna(subset=['status_date'])
-
-    for col in ["Legislative Goal", "Policy Impact Areas", "Key Provisions",
-                "Intended Beneficiaries", "Potential Impact", "description"]:
-        df[col] = df[col].fillna("")
-
-    df["combined_text"] = (
-        "Legislative Goal: " + df["Legislative Goal"] + "\n" +
-        "Policy Impact Areas: " + df["Policy Impact Areas"] + "\n" +
-        "Key Provisions: " + df["Key Provisions"] + "\n" +
-        "Intended Beneficiaries: " + df["Intended Beneficiaries"] + "\n" +
-        "Potential Impact: " + df["Potential Impact"] + "\n" +
-        "Description: " + df["description"]
-    )
-
-    return df
-
-@st.cache_resource
-def load_models():
-    embed_model = SentenceTransformer('all-MiniLM-L6-v2')
-    summarizer = pipeline("summarization", model="facebook/bart-large-cnn", tokenizer="facebook/bart-large-cnn")
-    return embed_model, summarizer
-
-@st.cache_data
-def compute_embeddings(texts, _model):
-    return _model.encode(texts, show_progress_bar=True)
-
-def rag_summarize(texts, summarizer, top_k=10):
-    if not texts:
-        return "No relevant content to summarize."
-    vect = TfidfVectorizer()
-    m = vect.fit_transform(texts)
-    mean_vec = m.mean(axis=0).A
-    scores = cosine_similarity(mean_vec, m).flatten()
-    top_indices = scores.argsort()[::-1][:top_k]
-    ctx = "\n".join(texts[i] for i in top_indices)
-    prompt = "summarize: " + ctx[:1024]
-    out = summarizer(prompt, max_length=300, min_length=100, do_sample=False)  
-    return out[0]['summary_text']
-
-def extract_month_year(q):
-    month_map = {m: i for i, m in enumerate(
-        ["january", "february", "march", "april", "may", "june",
-         "july", "august", "september", "october", "november", "december"], 1)}
-    ql = q.lower()
-    mon = next((v for k, v in month_map.items() if k in ql), None)
-    ym = re.search(r"(19|20)\d{2}", q)
-    yr = int(ym.group()) if ym else None
-    return mon, yr
-
-def extract_date_range(query):
-    month_map = {
-        "january": 1, "february": 2, "march": 3, "april": 4, "may": 5, "june": 6,
-        "july": 7, "august": 8, "september": 9, "october": 10, "november": 11, "december": 12
-    }
-
-    patterns = [
-        r"(?i)(?:from|between)?\s*([a-zA-Z]+)\s+(\d{4})\s*(?:to|through|and|-)\s*([a-zA-Z]+)\s+(\d{4})",
-    ]
-
-    for pattern in patterns:
-        match = re.search(pattern, query)
-        if match:
-            start_month_str, start_year = match.group(1).lower(), int(match.group(2))
-            end_month_str, end_year = match.group(3).lower(), int(match.group(4))
-
-            start_month = month_map.get(start_month_str)
-            end_month = month_map.get(end_month_str)
-
-            if start_month and end_month:
-                start_date = datetime(start_year, start_month, 1)
-                end_date = datetime(end_year, end_month, 28)
-                return start_date, end_date
-
-    return None, None
-
-def extract_topic_match(query, df):
-    query_lower = query.lower()
-    return df[
-        df['Category & Subcategory'].fillna('').str.lower().str.contains(query_lower) |
-        df['Intent'].fillna('').str.lower().str.contains(query_lower) |
-        df['Legislative Goal'].fillna('').str.lower().str.contains(query_lower) |
-        df['Policy Impact Areas'].fillna('').str.lower().str.contains(query_lower) |
-        df['Key Provisions'].fillna('').str.lower().str.contains(query_lower) |
-        df['Potential Impact'].fillna('').str.lower().str.contains(query_lower)
-    ]
-
-
-st.set_page_config(page_title="IL Legislative Trends Q&A", layout="wide")
-st.title("Illinois Legislative Trends Q&A")
-st.markdown("Ask about trends in topics like higher education, funding, etc.")
-
-df = load_data()
-embed_model, summarizer = load_models()
-
-query = st.text_input("Ask a question (e.g., ‘Trends from Jan 2024 to May 2025’):")
-
-if query:
-    start_date, end_date = extract_date_range(query)
-    df2 = extract_topic_match(query, df)
-
-    if df2.empty:
-        df2 = df
-
-    if start_date and end_date:
-        df2 = df2[(df2['status_date'] >= start_date) & (df2['status_date'] <= end_date)]
-        st.info(f"Filtering between: **{start_date:%B %Y}** and **{end_date:%B %Y}**")
-    else:
-        mon, yr = extract_month_year(query)
-        if yr:
-            df2 = df2[df2['status_date'].dt.year == yr]
-            if mon:
-                df2 = df2[df2['status_date'].dt.month == mon]
-                st.info(f"Filtering by date: **{datetime(yr, mon, 1):%B %Y}**")
-            else:
-                st.info(f"Filtering by year: **{yr}**")
-
-    if df2.empty:
-        st.warning("No matching records found.")
-    else:
-        st.subheader("Top Matching Bills")
-        for _, row in df2.iterrows():
-            date = row['status_date'].date()
-            bill_number = row['bill_number']
-            full_url = row['url']
-            cat = row.get('Category & Subcategory', '')
-            bene = row.get('Intended Beneficiaries', '')
-            goal = row.get('Legislative Goal', '')
-            impact = row.get('Policy Impact Areas', '')
-            provision = row.get('Key Provisions', '')
-            intent = row.get('Intent', '')
-            stance = row.get('Stance', '')
-            description = row.get('description', '')
-
-            st.markdown(f"**Date:** {date} | **Bill Number:** {bill_number}")
-            st.markdown(f"**Category:** {cat}")
-            st.markdown(f"**Intended Beneficiaries:** {bene}")
-            st.markdown(f"**Goal:** {goal}")
-            st.markdown(f"**Intent:** {intent} | **Stance:** {stance}")
-            st.markdown(f"**Policy Impact Area:** {impact}")
-            st.markdown(f"**Key Provision:** {provision}")
-            st.markdown(f"**Description:** {description}")
-            st.markdown(f"[View Full Bill Text]({full_url})\n")
-            st.divider()
-
-        st.subheader("RAG-Generated Overall Summary of All Matching Bills")
-        all_texts = df2['combined_text'].tolist()
-        summary = rag_summarize(all_texts, summarizer, top_k=15)
-        st.success(summary)