Update src/streamlit_app.py

src/streamlit_app.py

@@ -36,24 +36,45 @@ st.markdown(
 )
 
 # ── Data loading ──────────────────────────────────────────────────────────────
-@st.cache_data(show_spinner="Loading Chicago crime data…")
+@st.cache_data(show_spinner="Loading Chicago crime data (full dataset)…")
 def load_crime_data():
-    url = (
-        "https://data.cityofchicago.org/resource/ijzp-q8t2.json"
-        "?$where=year=2026"
-        "&$limit=10000"
-        "&$order=date%20DESC"
-    )
-    try:
-        df = pd.read_json(url)
-    except Exception as e:
-        st.error(f"Failed to load crime data: {e}")
+    """
+    Fetch ALL 2026 records in batches of 50 000.
+    Bar charts, line charts, and heatmaps use the full DataFrame.
+    Only map geo-points are sampled at render time.
+    """
+    all_chunks = []
+    limit  = 50000
+    offset = 0
+    while True:
+        url = (
+            "https://data.cityofchicago.org/resource/ijzp-q8t2.json"
+            "?$where=year=2026"
+            f"&$limit={limit}"
+            f"&$offset={offset}"
+            "&$order=date%20DESC"
+        )
+        try:
+            chunk = pd.read_json(url)
+        except Exception as e:
+            st.error(f"Failed to load crime data at offset {offset}: {e}")
+            break
+        if chunk.empty:
+            break
+        all_chunks.append(chunk)
+        if len(chunk) < limit:
+            break
+        offset += limit
+
+    if not all_chunks:
         return pd.DataFrame()
 
+    df = pd.concat(all_chunks, ignore_index=True)
+
     df["date"] = pd.to_datetime(df["date"], errors="coerce")
     for col in ["latitude", "longitude"]:
         df[col] = pd.to_numeric(df.get(col, pd.Series(dtype=float)), errors="coerce")
-    df = df.dropna(subset=["latitude", "longitude", "date"])
+    df = df.dropna(subset=["date"])          # keep rows even if coords missing; drop only for map
 
     df["Date_Only"] = df["date"].dt.floor("d")
     df["Hour"]      = df["date"].dt.hour
@@ -77,7 +98,6 @@ def load_crime_data():
     if "community_area" not in df.columns:
         df["community_area"] = None
 
-    # Period column
     def get_period(hour):
         if 6 < hour <= 12:
             return "Morning (6am-12pm)"
@@ -129,8 +149,13 @@ if df.empty:
     st.error("⚠️ Crime data could not be loaded. Please check the Chicago Data Portal.")
     st.stop()
 
+# Geo-only subset for map points (needs valid coords)
+df_geo = df.dropna(subset=["latitude", "longitude"]).copy()
+
+st.info(f"✅ Loaded **{len(df):,}** crime records for 2026 ({len(df_geo):,} with coordinates).")
+
 # ═══════════════════════════════════════════════════════════════════════════════
-# SECTION 1 — Linked dashboard (map | bar chart) & time-of-day line chart
+# SECTION 1 — Linked dashboard
 # ═══════════════════════════════════════════════════════════════════════════════
 st.markdown("---")
 st.header("🗺️ Interactive Crime Dashboard")
@@ -142,17 +167,19 @@ st.markdown(
     You can also **click a crime category** in the bar chart to drill into its temporal trend.
 
     The bottom line chart breaks daily incident counts into four time-of-day periods
-    (plus a total), so you can see not just *where* crime happens but *when* it peaks.
+    (plus a grey total line), so you can see not just *where* crime happens but *when* it peaks.
+    The bar chart and line chart use the **full dataset** — no sampling.
     """
 )
 
-# Altair selections (note: cross-chart filtering via selections only works when
-# the entire compound chart is rendered as one Altair object, which st.altair_chart supports)
 brush      = alt.selection_interval(name="brush")
 click_type = alt.selection_point(fields=["Primary Type"], name="click_type")
 click_dist = alt.selection_point(fields=["District_Str"], name="click_dist")
 
-# ── Map layer ─────────────────────────────────────────────────────────────────
+# Map: geo points sampled to keep browser responsive; bar + line use full df
+MAP_SAMPLE = 5000
+df_map_sample = df_geo.sample(min(MAP_SAMPLE, len(df_geo)), random_state=42)
+
 background = (
     alt.Chart(districts)
     .mark_geoshape(stroke="black", strokeWidth=0.6)
@@ -166,7 +193,7 @@ background = (
 )
 
 geo_points = (
-    alt.Chart(df)
+    alt.Chart(df_map_sample)
     .mark_circle(size=5)
     .encode(
         longitude="longitude:Q",
@@ -192,10 +219,10 @@ geo_points = (
 
 map_layer = (background + geo_points).project(type="mercator").properties(
     width=420, height=450,
-    title="Chicago Crime Map (Brush to select area / Click district)",
+    title=f"Chicago Crime Map (map shows {MAP_SAMPLE:,} sampled points for performance)",
 )
 
-# ── Crime-type bar chart ──────────────────────────────────────────────────────
+# Bar chart — FULL df, filtered by brush + district selections
 type_chart = (
     alt.Chart(df)
     .mark_bar()
@@ -205,13 +232,13 @@ type_chart = (
         color=alt.condition(click_type, alt.value("steelblue"), alt.value("lightgray")),
         tooltip=["Primary Type:N", "count():Q"],
     )
-    .properties(width=300, height=450, title="Crime Types")
+    .properties(width=300, height=450, title="Crime Types (full dataset)")
     .add_params(click_type)
     .transform_filter(brush)
     .transform_filter(click_dist)
 )
 
-# ── Time-of-day line chart ────────────────────────────────────────────────────
+# Line chart — FULL df, filtered by all three selections
 period_order = [
     "Morning (6am-12pm)",
     "Afternoon (12pm-6pm)",
@@ -264,10 +291,9 @@ total_line = (
 
 line_chart = (total_line + period_lines).properties(
     width=760, height=220,
-    title="Daily Crime Trend by Time of Day",
+    title="Daily Crime Trend by Time of Day (full dataset)",
 ).resolve_scale(color="shared")
 
-# ── Compose full dashboard ────────────────────────────────────────────────────
 dashboard = ((map_layer | type_chart) & line_chart).resolve_scale(color="independent")
 st.altair_chart(dashboard, use_container_width=True)
 
@@ -279,13 +305,13 @@ st.header("🕐 When Do Crimes Happen in Chicago?")
 st.markdown(
     """
     Different crimes follow different schedules. Use the **dropdown below** to filter
-    the heatmap to a specific crime category — or leave it on *All* to see the
-    overall pattern. Each cell shows the total number of incidents at that
-    day-of-week × hour-of-day combination; darker red means more incidents.
+    the heatmap to a specific crime category — or leave it on *All* to see the overall
+    pattern. Each cell shows the total number of incidents at that day-of-week × hour-of-day
+    combination across the **full dataset**; darker red means more incidents.
 
     Across nearly every category, Friday and Saturday evenings (6 pm – midnight)
-    stand out as the most active windows, while the early morning hours (2–5 am)
-    are quietest — except for a few crime types that peak overnight.
+    stand out as the most active windows, while the early-morning hours (2–5 am)
+    are consistently quietest.
     """
 )
 
@@ -320,7 +346,7 @@ heatmap = (
         width=700, height=380,
         title=alt.TitleParams(
             text=f"Crime Heatmap — {selected_hm}",
-            subtitle="Select a crime type above to filter · Darker = more incidents",
+            subtitle="Select a crime type above to filter · Full dataset · Darker = more incidents",
             fontSize=14,
         ),
     )
@@ -375,8 +401,9 @@ with col3:
             .project(type="mercator")
             .properties(width=360, height=440, title="Chicago Poverty Rate by Community Area")
         )
+        # Overlay: sample only for rendering dots on the map (visual only, not analysis)
         crime_overlay = (
-            alt.Chart(df.sample(min(5000, len(df)), random_state=42))
+            alt.Chart(df_geo.sample(min(5000, len(df_geo)), random_state=42))
             .mark_circle(size=3, color="steelblue", opacity=0.3)
             .encode(longitude="longitude:Q", latitude="latitude:Q")
         )
@@ -386,6 +413,7 @@ with col3:
 
 with col4:
     if not df_socio.empty and df["community_area"].notna().any():
+        # Crime count by community uses the FULL df — no sampling
         df_crime_count = (
             df.dropna(subset=["community_area"])
             .groupby("community_area").size()
@@ -426,7 +454,7 @@ with col4:
                     width=360, height=440,
                     title=alt.TitleParams(
                         text="Higher Poverty → More Crimes?",
-                        subtitle="Each dot = one community area  |  Dashed = trend",
+                        subtitle="Each dot = one community area  |  Dashed = trend  |  Full dataset counts",
                         fontSize=13,
                     ),
                 ),