Update streamlit_app.py

streamlit_app.py

@@ -19,6 +19,16 @@ API_BASE_URL = os.getenv(
 )
 
 
+def ensure_columns(df: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
+    df = df.copy()
+
+    for column in columns:
+        if column not in df.columns:
+            df[column] = None
+
+    return df
+
+
 @st.cache_data(ttl=300)
 def load_classified_articles() -> pd.DataFrame:
     try:
@@ -35,8 +45,31 @@ def load_classified_articles() -> pd.DataFrame:
         if df.empty:
             return df
 
-        df["published_at"] = pd.to_datetime(df.get("published_at"), errors="coerce", utc=True)
-        df["classified_at"] = pd.to_datetime(df.get("classified_at"), errors="coerce", utc=True)
+        df = ensure_columns(
+            df,
+            [
+                "article_id",
+                "title",
+                "description",
+                "source",
+                "label",
+                "raw_label",
+                "url",
+                "published_at",
+                "classified_at",
+            ],
+        )
+
+        df["published_at"] = pd.to_datetime(
+            df["published_at"],
+            errors="coerce",
+            utc=True,
+        )
+        df["classified_at"] = pd.to_datetime(
+            df["classified_at"],
+            errors="coerce",
+            utc=True,
+        )
 
         df["published_date"] = df["published_at"].dt.date
         df["published_day"] = df["published_at"].dt.strftime("%Y-%m-%d")
@@ -68,7 +101,7 @@ def load_daily_summary() -> dict:
 def normalize_summary_payload(summary: dict) -> dict:
     normalized = dict(summary)
 
-    nested_summary = summary.get("summary_json") or summary.get("top_stories")
+    nested_summary = summary.get("summary_json")
 
     if isinstance(nested_summary, str):
         try:
@@ -78,7 +111,7 @@ def normalize_summary_payload(summary: dict) -> dict:
         except Exception:
             pass
 
-    elif isinstance(nested_summary, dict) and "top_stories" in nested_summary:
+    elif isinstance(nested_summary, dict):
         normalized.update(nested_summary)
 
     normalized["executive_summary"] = (
@@ -108,8 +141,8 @@ def normalize_summary_payload(summary: dict) -> dict:
 def apply_filters(df: pd.DataFrame) -> pd.DataFrame:
     st.sidebar.header("Filters")
 
-    label_options = sorted(df["label"].dropna().unique().tolist()) if "label" in df else []
-    source_options = sorted(df["source"].dropna().unique().tolist()) if "source" in df else []
+    label_options = sorted(df["label"].dropna().unique().tolist())
+    source_options = sorted(df["source"].dropna().unique().tolist())
 
     default_labels = [
         label
@@ -129,8 +162,8 @@ def apply_filters(df: pd.DataFrame) -> pd.DataFrame:
         default=[],
     )
 
-    min_date = df["published_date"].min() if "published_date" in df and not df.empty else None
-    max_date = df["published_date"].max() if "published_date" in df and not df.empty else None
+    min_date = df["published_date"].min() if not df.empty else None
+    max_date = df["published_date"].max() if not df.empty else None
 
     date_range = None
     if min_date and max_date:
@@ -185,8 +218,8 @@ def render_metrics(df: pd.DataFrame, filtered_df: pd.DataFrame) -> None:
 
     c1.metric("Articles", len(df))
     c2.metric("Shown", len(filtered_df))
-    c3.metric("Sources", df["source"].nunique() if "source" in df else 0)
-    c4.metric("Categories", df["label"].nunique() if "label" in df else 0)
+    c3.metric("Sources", df["source"].nunique())
+    c4.metric("Categories", df["label"].nunique())
 
 
 def render_bullet_list(items: list[str], empty_message: str) -> None:
@@ -198,6 +231,30 @@ def render_bullet_list(items: list[str], empty_message: str) -> None:
         st.markdown(f"- {item}")
 
 
+def render_daily_summary_source_basis(
+    df: pd.DataFrame,
+    summary: dict,
+) -> pd.DataFrame:
+    summary_date = summary.get("summary_date")
+
+    if summary_date and "published_day" in df:
+        summary_df = df[df["published_day"] == summary_date]
+    else:
+        summary_df = df
+
+    if summary_date:
+        st.caption(
+            f"Summary is based on {len(summary_df)} classified articles "
+            f"published on {summary_date}."
+        )
+    else:
+        st.caption(
+            f"Summary is based on {len(summary_df)} classified articles."
+        )
+
+    return summary_df
+
+
 def render_daily_summary(summary: dict) -> None:
     st.subheader("Daily AI Summary")
 
@@ -257,7 +314,11 @@ def render_daily_summary(summary: dict) -> None:
         article_id = story.get("article_id")
 
         if pd.notnull(published_at):
-            published_at = pd.to_datetime(published_at, errors="coerce", utc=True)
+            published_at = pd.to_datetime(
+                published_at,
+                errors="coerce",
+                utc=True,
+            )
 
             if pd.notnull(published_at):
                 published_at = published_at.strftime("%Y-%m-%d %H:%M UTC")
@@ -285,7 +346,7 @@ def render_daily_summary(summary: dict) -> None:
                 st.write(decision_relevance)
 
             if url:
-                st.markdown(f"[Open article]({url})")
+                st.link_button("Open article", url)
 
             if article_id:
                 st.caption(f"Article ID: {article_id}")
@@ -316,9 +377,15 @@ def render_article_browser(df: pd.DataFrame) -> None:
     elif sort_option == "Oldest first":
         display_df = display_df.sort_values("published_at", ascending=True)
     elif sort_option == "Action category":
-        display_df = display_df.sort_values(["label", "published_at"], ascending=[True, False])
+        display_df = display_df.sort_values(
+            ["label", "published_at"],
+            ascending=[True, False],
+        )
     elif sort_option == "Source":
-        display_df = display_df.sort_values(["source", "published_at"], ascending=[True, False])
+        display_df = display_df.sort_values(
+            ["source", "published_at"],
+            ascending=[True, False],
+        )
 
     max_rows = st.slider("Number of articles to display", 5, 100, 20)
     display_df = display_df.head(max_rows)
@@ -345,7 +412,7 @@ def render_article_browser(df: pd.DataFrame) -> None:
 
             url = row.get("url")
             if pd.notnull(url) and str(url).strip():
-                st.markdown(f"[Open article]({url})")
+                st.link_button("Open article", str(url))
 
             st.markdown("**More details**")
 
@@ -365,11 +432,18 @@ def main() -> None:
         "with filters for action categories, dates, sources, and search terms."
     )
 
+    if st.sidebar.button("Refresh data"):
+        st.cache_data.clear()
+        st.rerun()
+
     df = load_classified_articles()
     summary = load_daily_summary()
 
     if df.empty:
-        st.warning("No classified articles found yet. Check whether the API is live and returning data.")
+        st.warning(
+            "No classified articles found yet. "
+            "Check whether the API is live and returning data."
+        )
         return
 
     section = st.segmented_control(
@@ -379,7 +453,8 @@ def main() -> None:
     )
 
     if section == "Daily Summary":
-        render_metrics(df, df)
+        summary_df = render_daily_summary_source_basis(df, summary)
+        render_metrics(df, summary_df)
         render_daily_summary(summary)
 
     elif section == "Articles":