Deploy from GitHub Actions

analysis.py

@@ -1,9 +1,11 @@
+import math
 import sys
 from pathlib import Path
 
 import altair as alt
 import contextily as ctx
 import geopandas as gpd
+import matplotlib.dates as mdates
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
@@ -1554,3 +1556,195 @@ def generate_seasonal_plot(data, year, shapefile_path):
         wbids=wbids,
         reporting_end_month=st.session_state.reporting_month,
     )
+
+
+def plot_do_timeseries(
+    df: pd.DataFrame,
+    period: str = "Yearly",
+    sector: str = "All",
+    epa_thresh: float = 4.8,
+) -> Figure:
+    """
+    Create a time series plot of dissolved oxygen levels for surface and bottom measurements.
+
+    Reference:
+    https://www.hudsonriver.org/ccmp/soe/water-quality/do
+
+    Parameters:
+    -----------
+    df : pd.DataFrame
+        Filtered dataframe containing dissolved oxygen measurements
+    period : str
+        'yearly' or 'monthly' aggregation period
+    epa_thresh : float
+        EPA threshold value for DO in mg/L
+
+    Returns:
+    --------
+    Figure
+        Matplotlib figure containing the plot
+    """
+    period = period.lower()
+    # Filter for DO data and pivot for surface/bottom
+    do_data = df[
+        (df["Org_Analyte_Name"] == "Dissolved Oxygen")
+        & (df["Sample_Position"].isin(["Surface", "Bottom"]))
+    ].copy()
+
+    # Create time grouping based on period
+    if period == "yearly":
+        do_data["Period"] = do_data["Reporting_Year"]
+    else:  # monthly
+        do_data["Period"] = pd.to_datetime(
+            do_data["Activity_Start_Date_Time"]
+        ).dt.to_period("M")
+        do_data["Period_Start"] = do_data["Period"].dt.to_timestamp()
+
+    # Calculate means for each position and period
+    means = (
+        do_data.groupby(["Period", "Sample_Position"], observed=True)[
+            "Org_Result_Value"
+        ]
+        .mean()
+        .reset_index()
+        .pivot(index="Period", columns="Sample_Position", values="Org_Result_Value")
+    )
+
+    # Create figure
+    fig, ax = plt.subplots(figsize=(15, 8))
+
+    # Convert Period index to proper format for plotting
+    if period == "yearly":
+        x_values = np.array(means.index.astype(float))  # Explicitly create numpy array
+    else:
+        # Convert to numpy array of datetime64
+        x_values = np.array(
+            [pd.Period(idx).to_timestamp() for idx in means.index],
+            dtype="datetime64[ns]",
+        )
+
+    # Plot connecting lines only (no markers)
+    for i, (idx, row) in enumerate(means.iterrows()):
+        x_val = x_values[i]
+        ax.plot(
+            [x_val, x_val],  # Use scalar value instead of list
+            [row["Bottom"], row["Surface"]],
+            color="lightgray",
+            linewidth=1,
+            zorder=1,
+            solid_capstyle="round",
+        )
+
+    # Calculate dynamic point size based on number of points
+    n_points = len(x_values)
+    base_size = 80  # Maximum point size
+    min_size = 20  # Minimum point size
+
+    # Exponential decay formula: size decreases as number of points increases
+    point_size = max(
+        min_size,
+        base_size * math.exp(-0.0015 * n_points),
+    )
+    # Update scatter plot styling
+    surface_scatter = ax.scatter(
+        x_values,
+        means["Surface"],
+        color="#5AA4D8",
+        s=point_size,
+        zorder=2,
+        label="Surface",
+        edgecolors="white",
+        linewidth=1,
+        alpha=0.9,
+    )
+    bottom_scatter = ax.scatter(
+        x_values,
+        means["Bottom"],
+        color="#1B4B8A",
+        s=point_size,
+        zorder=2,
+        label="Bottom",
+        edgecolors="white",
+        linewidth=1,
+        alpha=0.9,
+    )
+
+    # Update EPA threshold line
+    threshold_line = ax.axhline(
+        y=epa_thresh,
+        color="#FF8C00",
+        linestyle="--",
+        alpha=0.9,
+        linewidth=1,
+        label=f"EPA threshold: {epa_thresh} mg/L",
+        zorder=0,
+    )
+
+    # Customize legend
+    ax.legend(
+        handles=[surface_scatter, bottom_scatter, threshold_line],
+        loc="upper right",
+        frameon=False,
+        ncol=1,  # Stack legend items vertically
+        bbox_to_anchor=(1.0, 1.0),  # Position at top right
+        handletextpad=0.5,  # Reduce space between handle and text
+    )
+
+    # Customize spines - only show bottom spine
+    ax.spines["top"].set_visible(False)
+    ax.spines["right"].set_visible(False)
+    ax.spines["left"].set_visible(False)
+    ax.spines["bottom"].set_color("black")
+    ax.spines["bottom"].set_linewidth(0.5)
+
+    # Customize plot with modified grid and axis settings
+    ax.set_xlabel("Year" if period == "yearly" else "Month")
+    ax.set_ylabel("Dissolved Oxygen (mg/L)")
+    ax.set_title("Long-term Dissolved Oxygen Trends")
+    ax.grid(True, axis="y", alpha=0.15, linestyle="-", color="gray")
+
+    # Set y-axis limits with some padding
+    ymin = max(int(min(means["Bottom"].min(), epa_thresh) * 0.9) - 1, 0)
+    # ymin = 0
+    ymax = means["Surface"].max() * 1.1
+    ax.set_ylim(ymin, ymax)
+    yticks = np.arange(ymin, ymax, 2)
+    ax.set_yticks(yticks)
+
+    # Remove tick marks but keep labels
+    ax.tick_params(axis="y", which="both", length=0)  # Remove y-axis tick marks
+
+    # Adjust x-axis ticks and limits
+    if period == "monthly":
+        ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y"))
+        ax.xaxis.set_major_locator(mdates.YearLocator())
+        plt.xticks(rotation=0)
+
+        # Convert to datetime for padding
+        start_date = mdates.date2num(
+            pd.Timestamp(min(x_values)) - pd.DateOffset(months=1)
+        )
+        end_date = mdates.date2num(
+            pd.Timestamp(max(x_values)) + pd.DateOffset(months=1)
+        )
+        ax.set_xlim(mdates.num2date(start_date), mdates.num2date(end_date))
+    else:
+        # For yearly data, ensure whole number ticks but month-based padding
+        min_year = float(np.floor(min(x_values)))
+        max_year = float(np.ceil(max(x_values)))
+
+        # Set whole number ticks
+        years = np.arange(min_year, max_year + 1)
+        ax.set_xticks(years)
+
+        # Set limits with one month padding
+        ax.set_xlim(
+            min_year - 0.083, max_year + 0.083
+        )  # ~1/12 of a year for month padding
+
+    # Move y-axis labels to the left of the gridlines
+    ax.yaxis.tick_left()
+    ax.yaxis.set_label_position("left")
+
+    plt.tight_layout()
+    return fig

app.py

@@ -5,6 +5,7 @@ from config import AppConfig
 from dashboard_analytics import render_analytics_page
 from pages import (
     calendar_heatmaps_page,
+    dissolved_oxygen_page,
     do_temp_relationship_page,
     home_page,
     nutrient_ratios_page,
@@ -48,6 +49,7 @@ page_dict["Water Quality Portal"] = [
     calendar_heatmaps_page,
     seasonal_trends_page,
     nutrient_ratios_page,
+    dissolved_oxygen_page,
     do_temp_relationship_page,
     raw_data_page,
     settings_page,

pages.py

@@ -551,7 +551,7 @@ def seasonal_trends_section():
     # Move filters to sidebar
     st.sidebar.markdown("### Filter Options")
     analyte = st.sidebar.selectbox(
-        "Select Analyte:", ["Salinity"], index=0, key="seasonal_analyte_select"
+        "Select Parameter:", ["Salinity"], index=0, key="seasonal_analyte_select"
     )
     selected_year = st.sidebar.selectbox(
         "Select Year:", sorted(years, reverse=True), index=0, key="seasonal_year_select"
@@ -906,3 +906,8 @@ nutrient_ratios_page = st.Page(
     icon=":material/scatter_plot:",
 )
 settings_page = st.Page("settings.py", title="Settings", icon=":material/settings:")
+dissolved_oxygen_page = st.Page(
+    "ui/pages/dissolved_oxygen.py",
+    title="Dissolved Oxygen",
+    icon=":material/water_drop:",
+)

ui/pages/dissolved_oxygen.py

@@ -0,0 +1,31 @@
+import streamlit as st
+
+from analysis import plot_do_timeseries
+
+
+def dissolved_oxygen():
+    # Add period selector to sidebar
+    period = st.sidebar.radio(
+        "Time Period", options=["Monthly", "Yearly"], key="do_period"
+    )
+    sector_options = ["All"] + list(st.session_state.data["raw_df"]["Sector"].unique())
+    sector = st.sidebar.selectbox("Sector", options=sector_options, key="do_sector")
+    if sector == "All":
+        filtered_df = st.session_state.data["raw_df"]
+    else:
+        filtered_df = st.session_state.data["raw_df"][
+            st.session_state.data["raw_df"]["Sector"] == sector
+        ]
+
+    return plot_do_timeseries(filtered_df, period=period)
+
+
+st.info(
+    """
+    This is a half-baked attempt to replicate the Hudson River Estuary Program's excellent
+    [Long Term Findings chart for dissolved oxygen](https://www.hudsonriver.org/ccmp/soe/water-quality/do).
+    The EPA threshold is copied from the reference image and not specific to the water quality standards for our area.
+    """
+)
+fig = dissolved_oxygen()
+st.pyplot(fig)

utils/data_loading.py

@@ -114,7 +114,7 @@ class DataManager:
             return self._get_empty_data_structure()
 
 
-@timer(include_params=False)
+@timer(include_params=True)
 def get_raw_data(file_path: str) -> pd.DataFrame:
     """Load raw data from parquet file"""
     return pd.read_parquet(file_path)
@@ -200,6 +200,7 @@ def add_lat_long(raw_df: pd.DataFrame, stations_df: pd.DataFrame) -> pd.DataFram
     return raw_df.drop("Number", axis=1)
 
 
+@timer(include_params=False)
 def get_stations_data() -> pd.DataFrame:
     """
     Return stations data as a dataframe with the most recent and earliest sample dates for each station.