Add learned wait-time estimates and stretch chart to fill frame

Timing estimates (backend):
- config.py: add FPPMW_CUTOVER_DATE, TIMING_MIN_SAMPLES,
TIMING_DEFAULT_CURRENT_SEC / ARCHIVE_SEC as named constants
- analytics.py: migrate schema to add duration_ms + source_type columns;
extend log_request() to accept them; add get_timing_estimate() which
returns p75 of the last 100 successful view durations per source_type,
falling back to hardcoded defaults until >= TIMING_MIN_SAMPLES records exist
- api.py: time fetch + process pipeline in all three data handlers;
classify each request as 'current' (>= 2026-01-11) or 'archive';
pass duration_ms + source_type to log_request; expose get_timing_estimate
results and cutover_date in /api/info

Dynamic loading message (frontend):
- app.js: store cutover_date and estimates from /api/info on boot;
add formatEstimate() helper that converts seconds to human-readable bands
and appends sample count when the estimate is learned (not default);
replace hardcoded loading strings with the dynamic estimate message

Chart sizing:
- style.css: widen main max-width 1200px → 1600px so the chart fills
more of the viewport on wide screens; change chart height to
clamp(400px, 50vh, 650px) for proportional scaling across screen sizes
- app.js: add autosize: true to Plotly layout so the chart fills its
container on first render without waiting for a resize event

https://claude.ai/code/session_01TrCrzZmWbQscJiAQaW7Rg8

app/config.py

@@ -8,6 +8,12 @@ AEMO_ARCHIVE_URL = "https://nemweb.com.au/Reports/Archive/FPPDAILY/"
 # Data availability: first FPPMW SCADA bundle is PUBLIC_NEXT_DAY_FPPMW_20250228.zip
 DATA_START_DATE = date(2025, 2, 28)
 
+# From this date onward, data is served as individual daily ZIPs from the
+# Current directory (faster).  Before this date, large archive bundles are
+# used (slower).  This cutover is also sent to the frontend so both sides
+# classify requests consistently.
+FPPMW_CUTOVER_DATE = date(2026, 1, 11)
+
 # Request limits
 MAX_DAYS_PER_REQUEST = 1
 
@@ -19,6 +25,12 @@ AEMO_READ_TIMEOUT = 60
 ANALYTICS_TOKEN = os.environ.get("ANALYTICS_TOKEN", "changeme")
 ANALYTICS_DB_PATH = "/tmp/analytics.db"
 
+# Timing estimates: minimum successful samples before learned estimate is used
+# instead of the hardcoded default.
+TIMING_MIN_SAMPLES = 5
+TIMING_DEFAULT_CURRENT_SEC = 120   # ~2 min default for current-dir files
+TIMING_DEFAULT_ARCHIVE_SEC  = 480  # ~8 min default for archive bundles
+
 # Columns to keep from the raw CSV (unit identifier used only for filtering,
 # not included in output)
 REQUIRED_COLUMNS = [

app/routers/api.py

@@ -3,6 +3,7 @@ REST API endpoints for BESS SCADA data.
 """
 import json
 import logging
+import time
 from datetime import date, datetime
 from pathlib import Path
 
@@ -11,9 +12,14 @@ logger = logging.getLogger(__name__)
 from fastapi import APIRouter, HTTPException, Query, Request
 from fastapi.responses import Response
 
-from app.config import ANALYTICS_TOKEN, DATA_START_DATE, MAX_DAYS_PER_REQUEST
+from app.config import (
+    ANALYTICS_TOKEN,
+    DATA_START_DATE,
+    FPPMW_CUTOVER_DATE,
+    MAX_DAYS_PER_REQUEST,
+)
 from app.services.aemo_fetcher import AEMOFetchError, fetch_csv_for_date
-from app.services.analytics import get_stats, log_request
+from app.services.analytics import get_stats, get_timing_estimate, log_request
 from app.services.gen_info_fetcher import fetch_bess_list
 from app.services.data_processor import (
     DataProcessingError,
@@ -43,6 +49,11 @@ def _parse_date(date_str: str) -> date:
         raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD.")
 
 
+def _source_type(target_date: date) -> str:
+    """'current' for dates served from the Current directory, 'archive' otherwise."""
+    return "current" if target_date >= FPPMW_CUTOVER_DATE else "archive"
+
+
 @router.get("/bess")
 async def get_bess_list():
     """Return in-service battery storage units grouped by state, sourced from the
@@ -81,13 +92,16 @@ async def get_data(
     """
     target_date = _parse_date(date)
     ip = _get_ip(request)
+    src = _source_type(target_date)
 
+    t0 = time.monotonic()
     try:
         csv_bytes = await _fetch_day_csv(target_date, duid)
         df = filter_and_process(csv_bytes, duid, target_date)
+        duration_ms = int((time.monotonic() - t0) * 1000)
         summary = compute_summary(df)
         records = to_json_records(df)
-        log_request(ip, duid, date, "view")
+        log_request(ip, duid, date, "view", duration_ms=duration_ms, source_type=src)
         return {
             "duid": duid,
             "date": date,
@@ -111,12 +125,15 @@ async def download_csv(
     """Download full filtered data as CSV."""
     target_date = _parse_date(date)
     ip = _get_ip(request)
+    src = _source_type(target_date)
 
+    t0 = time.monotonic()
     try:
         csv_bytes = await _fetch_day_csv(target_date, duid)
         df = filter_and_process(csv_bytes, duid, target_date)
+        duration_ms = int((time.monotonic() - t0) * 1000)
         output = to_csv_bytes(df)
-        log_request(ip, duid, date, "download_csv")
+        log_request(ip, duid, date, "download_csv", duration_ms=duration_ms, source_type=src)
         filename = f"BESS_SCADA_{duid}_{date}.csv"
         return Response(
             content=output,
@@ -138,12 +155,15 @@ async def download_parquet(
     """Download full filtered data as Parquet."""
     target_date = _parse_date(date)
     ip = _get_ip(request)
+    src = _source_type(target_date)
 
+    t0 = time.monotonic()
     try:
         csv_bytes = await _fetch_day_csv(target_date, duid)
         df = filter_and_process(csv_bytes, duid, target_date)
+        duration_ms = int((time.monotonic() - t0) * 1000)
         output = to_parquet_bytes(df)
-        log_request(ip, duid, date, "download_parquet")
+        log_request(ip, duid, date, "download_parquet", duration_ms=duration_ms, source_type=src)
         filename = f"BESS_SCADA_{duid}_{date}.parquet"
         return Response(
             content=output,
@@ -169,5 +189,10 @@ def info():
     """Return app metadata for the frontend."""
     return {
         "data_start_date": DATA_START_DATE.isoformat(),
+        "cutover_date": FPPMW_CUTOVER_DATE.isoformat(),
         "max_days_per_request": MAX_DAYS_PER_REQUEST,
+        "estimates": {
+            "current": get_timing_estimate("current"),
+            "archive": get_timing_estimate("archive"),
+        },
     }

app/services/analytics.py

@@ -1,11 +1,16 @@
 """
 Lightweight SQLite-based analytics logger.
-Tracks request counts, IPs, and selected parameters.
+Tracks request counts, IPs, selected parameters, and end-to-end durations.
 """
 import sqlite3
 from datetime import datetime
 
-from app.config import ANALYTICS_DB_PATH
+from app.config import (
+    ANALYTICS_DB_PATH,
+    TIMING_DEFAULT_ARCHIVE_SEC,
+    TIMING_DEFAULT_CURRENT_SEC,
+    TIMING_MIN_SAMPLES,
+)
 
 
 def _get_conn() -> sqlite3.Connection:
@@ -15,34 +20,96 @@ def _get_conn() -> sqlite3.Connection:
 
 
 def init_db() -> None:
-    """Create the analytics table if it doesn't exist."""
+    """Create the analytics table and migrate any missing columns."""
     with _get_conn() as conn:
         conn.execute("""
             CREATE TABLE IF NOT EXISTS requests (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                timestamp TEXT NOT NULL,
-                ip TEXT,
-                duid TEXT,
-                date TEXT,
-                action TEXT
+                id          INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp   TEXT    NOT NULL,
+                ip          TEXT,
+                duid        TEXT,
+                date        TEXT,
+                action      TEXT,
+                duration_ms INTEGER,
+                source_type TEXT
             )
         """)
+        # Migrate pre-existing tables that lack the new columns.
+        existing = {row[1] for row in conn.execute("PRAGMA table_info(requests)")}
+        if "duration_ms" not in existing:
+            conn.execute("ALTER TABLE requests ADD COLUMN duration_ms INTEGER")
+        if "source_type" not in existing:
+            conn.execute("ALTER TABLE requests ADD COLUMN source_type TEXT")
         conn.commit()
 
 
-def log_request(ip: str, duid: str, date: str, action: str) -> None:
-    """Log a single request."""
+def log_request(
+    ip: str,
+    duid: str,
+    date: str,
+    action: str,
+    *,
+    duration_ms: int | None = None,
+    source_type: str | None = None,
+) -> None:
+    """Log a single request, optionally with timing and data-source type."""
     try:
         with _get_conn() as conn:
             conn.execute(
-                "INSERT INTO requests (timestamp, ip, duid, date, action) VALUES (?, ?, ?, ?, ?)",
-                (datetime.utcnow().isoformat(), ip, duid, date, action),
+                """
+                INSERT INTO requests
+                    (timestamp, ip, duid, date, action, duration_ms, source_type)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                """,
+                (datetime.utcnow().isoformat(), ip, duid, date, action,
+                 duration_ms, source_type),
             )
             conn.commit()
     except Exception:
         pass  # Never let analytics failures break the main flow
 
 
+def get_timing_estimate(source_type: str) -> dict:
+    """
+    Return a p75 wait-time estimate in seconds for the given source_type
+    ('current' or 'archive'), derived from the last 100 successful view
+    requests that recorded a duration.
+
+    Falls back to a hardcoded default when fewer than TIMING_MIN_SAMPLES
+    records are available.
+    """
+    default_sec = (
+        TIMING_DEFAULT_CURRENT_SEC
+        if source_type == "current"
+        else TIMING_DEFAULT_ARCHIVE_SEC
+    )
+    try:
+        with _get_conn() as conn:
+            rows = conn.execute(
+                """
+                SELECT duration_ms FROM requests
+                WHERE source_type = ?
+                  AND duration_ms IS NOT NULL
+                  AND action = 'view'
+                ORDER BY id DESC
+                LIMIT 100
+                """,
+                (source_type,),
+            ).fetchall()
+
+        durations = sorted(row[0] for row in rows)
+        n = len(durations)
+        if n < TIMING_MIN_SAMPLES:
+            return {"seconds": default_sec, "sample_count": n, "is_default": True}
+
+        p75_idx = int(0.75 * (n - 1))
+        p75_sec = round(durations[p75_idx] / 1000)
+        return {"seconds": p75_sec, "sample_count": n, "is_default": False}
+
+    except Exception:
+        return {"seconds": default_sec, "sample_count": 0, "is_default": True}
+
+
 def get_stats() -> dict:
     """Return summary analytics."""
     try:
@@ -58,7 +125,10 @@ def get_stats() -> dict:
                 "SELECT ip, COUNT(*) as cnt FROM requests GROUP BY ip ORDER BY cnt DESC LIMIT 50"
             ).fetchall()
             recent = conn.execute(
-                "SELECT timestamp, ip, duid, date, action FROM requests ORDER BY id DESC LIMIT 100"
+                """
+                SELECT timestamp, ip, duid, date, action, duration_ms, source_type
+                FROM requests ORDER BY id DESC LIMIT 100
+                """
             ).fetchall()
 
         return {

app/static/css/style.css

@@ -47,7 +47,7 @@ header p {
 }
 
 main {
-  max-width: 1200px;
+  max-width: 1600px;
   margin: 0 auto;
   padding: 2rem 1.5rem;
   display: flex;
@@ -233,7 +233,7 @@ select:disabled, input:disabled {
 /* ── Chart ── */
 #chart-container {
   width: 100%;
-  height: 400px;
+  height: clamp(400px, 50vh, 650px);
 }
 
 /* ── Download bar ── */

app/static/js/app.js

@@ -10,6 +10,11 @@ let qualityFlags = {};
 let currentData = null;
 let currentDuid = null;
 let currentDate = null;
+let cutoverDate  = '2026-01-11';   // updated from /api/info on boot
+let appEstimates = {               // updated from /api/info on boot
+  current: { seconds: 120, sample_count: 0, is_default: true },
+  archive: { seconds: 480, sample_count: 0, is_default: true },
+};
 
 /* ── DOM refs ── */
 const loadingMsg = document.getElementById('loading-msg');
@@ -36,10 +41,12 @@ async function init() {
   inpDate.value = yesterday.toISOString().slice(0, 10);
   inpDate.max   = yesterday.toISOString().slice(0, 10);
 
-  // Fetch app info for min date
+  // Fetch app info for min date, cutover date, and timing estimates
   try {
     const info = await fetch(`${API}/api/info`).then(r => r.json());
     inpDate.min = info.data_start_date;
+    if (info.cutover_date) cutoverDate = info.cutover_date;
+    if (info.estimates)    appEstimates = info.estimates;
   } catch (_) {}
 
   // Load BESS list and quality flags in parallel
@@ -92,6 +99,27 @@ function onBessChange() {
   btnLoad.disabled = !hasBess;
 }
 
+/* ── Timing estimate helpers ── */
+
+/**
+ * Format a timing estimate object into a human-readable string fragment.
+ * est = { seconds, sample_count, is_default }
+ */
+function formatEstimate(est) {
+  const sec = est.seconds || 120;
+  const suffix = est.is_default ? '' : ` (based on ${est.sample_count} recent request${est.sample_count === 1 ? '' : 's'})`;
+  let duration;
+  if (sec < 60)        duration = 'less than a minute';
+  else if (sec < 100)  duration = 'about 1 minute';
+  else if (sec < 160)  duration = 'about 2 minutes';
+  else if (sec < 220)  duration = 'about 3 minutes';
+  else if (sec < 310)  duration = 'about 4–5 minutes';
+  else if (sec < 420)  duration = 'about 6–7 minutes';
+  else if (sec < 570)  duration = 'about 8–10 minutes';
+  else                 duration = '10 minutes or more';
+  return `this typically takes ${duration}${suffix}`;
+}
+
 /* ── Load data ── */
 function setFormLocked(locked) {
   selState.disabled = locked;
@@ -109,15 +137,12 @@ async function onLoad() {
   hideError();
   hideResults();
 
-  // Give a realistic wait-time estimate based on how the NEMWEB archive is structured.
-  // Files from 11 Jan 2026 onward are served as individual daily ZIPs (~2–3 min).
-  // Older dates are packed in large monthly/quarterly archives (~5–10 min or more).
-  const RECENT_CUTOFF = '2026-01-11';
-  if (date >= RECENT_CUTOFF) {
-    loadingMsg.textContent = 'Fetching data from AEMO NEMWEB… this typically takes 2–3 minutes. Please wait.';
-  } else {
-    loadingMsg.textContent = 'Fetching data from AEMO NEMWEB… older dates are stored in large archive files and may take 5–10 minutes or more. Please wait.';
-  }
+  // Show a wait-time estimate learned from previous successful requests.
+  // The cutover date and p75 estimates are provided by /api/info on boot.
+  const srcType = date >= cutoverDate ? 'current' : 'archive';
+  const est     = appEstimates[srcType] || appEstimates.current;
+  loadingMsg.textContent =
+    `Fetching data from AEMO NEMWEB\u2026 ${formatEstimate(est)}. Please wait.`;
 
   showLoading(true);
   setFormLocked(true);
@@ -217,6 +242,7 @@ function renderChart(data) {
   });
 
   const layout = {
+    autosize: true,
     paper_bgcolor: 'transparent',
     plot_bgcolor:  'transparent',
     font: { color: '#e2e8f0', size: 11 },