Smart filter: scan full history for last N matching messages + keyword accuracy fixes

pages/comments.py

@@ -104,24 +104,61 @@ with _cf3:
 with _cf4:
     _search_term = st.text_input("Search messages", placeholder="Filter by keyword...")
 
-_filtered = _feed_df.copy() if not _feed_df.empty else pd.DataFrame()
-if not _filtered.empty:
-    if _sentiment_filter != "All":
-        _filtered = _filtered[_filtered["sentiment"] == _sentiment_filter]
-    if _topic_filter != "All":
-        _filtered = _filtered[_filtered["topic"] == _topic_filter]
-    if _action_type_filter != "All":
-        if "action_type" in _filtered.columns:
-            _filtered = _filtered[_filtered["action_type"] == _action_type_filter]
-    if _search_term:
-        _filtered = _filtered[_filtered["text"].str.contains(_search_term, case=False, na=False)]
+# ── Smart filtering: when a filter is active, scan full history
+# to find the last msg_limit matching messages instead of filtering
+# only within the current window.
+_any_filter = (
+    _sentiment_filter != "All"
+    or _topic_filter != "All"
+    or _action_type_filter != "All"
+    or bool(_search_term)
+)
+
+if _any_filter:
+    # Load full history for the feed key
+    _full_raw = load_stream_data(_feed_key)
+    if _full_raw:
+        _full_df = pd.DataFrame(_full_raw)
+        _full_df["sentiment"] = _full_df["sentiment"].apply(clean_sentiment)
+        _full_df["topic"]     = _full_df["topic"].apply(clean_topic) if "topic" in _full_df.columns else "General"
+        # Apply filters on full history
+        _filtered = _full_df.copy()
+        if _sentiment_filter != "All":
+            _filtered = _filtered[_filtered["sentiment"] == _sentiment_filter]
+        if _topic_filter != "All":
+            _filtered = _filtered[_filtered["topic"] == _topic_filter]
+        if _action_type_filter != "All":
+            if "action_type" in _filtered.columns:
+                _filtered = _filtered[_filtered["action_type"] == _action_type_filter]
+        if _search_term:
+            _filtered = _filtered[_filtered["text"].str.contains(_search_term, case=False, na=False)]
+        # Cap to last msg_limit matching results
+        if len(_filtered) > msg_limit:
+            _filtered = _filtered.iloc[-msg_limit:]
+    else:
+        _filtered = pd.DataFrame()
+    _total_matching = len(_filtered)
+    _total_scanned  = len(_full_raw) if _full_raw else 0
+else:
+    # No filter — just use the window
+    _filtered = _feed_df.copy() if not _feed_df.empty else pd.DataFrame()
+    _total_matching = len(_filtered)
+    _total_scanned  = len(_feed_df)
 
 _feed_hdr, _feed_dl = st.columns([3, 1])
 with _feed_hdr:
-    st.markdown(
-        f'<div style="font-size:0.78rem;color:var(--text-3);margin-bottom:12px;">Showing {len(_filtered)} of {len(_feed_df)} messages</div>',
-        unsafe_allow_html=True
-    )
+    if _any_filter:
+        st.markdown(
+            f'<div style="font-size:0.78rem;color:var(--text-3);margin-bottom:12px;">'
+            f'Showing {len(_filtered)} matching (last {msg_limit} from {_total_scanned} total)</div>',
+            unsafe_allow_html=True
+        )
+    else:
+        st.markdown(
+            f'<div style="font-size:0.78rem;color:var(--text-3);margin-bottom:12px;">'
+            f'Showing {len(_filtered)} of {len(_feed_df)} messages</div>',
+            unsafe_allow_html=True
+        )
 with _feed_dl:
     if not _filtered.empty:
         _export_cols = [c for c in ["author", "text", "sentiment", "confidence", "topic", "time"] if c in _filtered.columns]