feat: Enhance DWD metadata with date ranges and station availability, add 24h index expiry, remove parse date filtering, and improve solar plot with daily aggregation and dual axes.

funs/dwd_metadata.R

@@ -58,6 +58,13 @@ fetch_dwd_file_index <- function() {
                             }
 
                             if (length(zip_files) > 0) {
+                                # Extract dates if possible (format: _YYYYMMDD_YYYYMMDD_)
+                                # Pattern: _(17|18|19|20)\d{6}_(17|18|19|20)\d{6}
+                                date_pattern <- "_((?:17|18|19|20)\\d{6})_((?:17|18|19|20)\\d{6})"
+
+                                start_dates <- str_match(zip_files, date_pattern)[, 2]
+                                end_dates <- str_match(zip_files, date_pattern)[, 3]
+
                                 # Create index DataFrame
                                 df <- data.frame(
                                     id = ids,
@@ -65,6 +72,8 @@ fetch_dwd_file_index <- function() {
                                     url = paste0(url, zip_files), # Absolute URL
                                     param = param,
                                     type = subtype,
+                                    start_date = start_dates,
+                                    end_date = end_dates,
                                     stringsAsFactors = FALSE
                                 )
                                 index_list[[paste(param, subtype, sep = "_")]] <- df

funs/parse_dwd.R

@@ -100,13 +100,13 @@ read_dwd_data <- function(zip_path, start_date = NULL, end_date = NULL) {
             # Filter valid dates
             df <- df[!is.na(df$datetime), ]
 
-            # Window Filter
-            if (!is.null(start_date)) {
-                df <- df[df$datetime >= (as.POSIXct(start_date) - days(1)), ]
-            }
-            if (!is.null(end_date)) {
-                df <- df[df$datetime <= (as.POSIXct(end_date) + days(1)), ]
-            }
+            # Window Filter - REMOVED to allow caller to inspect full range
+            # if (!is.null(start_date)) {
+            #     df <- df[df$datetime >= (as.POSIXct(start_date) - days(1)), ]
+            # }
+            # if (!is.null(end_date)) {
+            #     df <- df[df$datetime <= (as.POSIXct(end_date) + days(1)), ]
+            # }
 
             # Column Mapping
             weather_cols <- c("temp", "rh", "precip", "wind_speed", "wind_dir", "pressure", "station_pressure", "cloud_cover", "wind_gust_max", "solar_global", "sunshine_duration")

funs/plot_weather_dwd.R

@@ -628,7 +628,7 @@ create_diurnal_plot <- function(df, offset_hours = 0) {
         plotly::config(displaylogo = FALSE)
 }
 
-#' Create Solar Plot (Preserved)
+#' Create Solar Plot (Enhanced - Daily Aggregation)
 create_solar_plot <- function(df) {
     if (is.null(df) || nrow(df) == 0) {
         return(create_empty_plot("No data available for the selected period"))
@@ -637,49 +637,105 @@ create_solar_plot <- function(df) {
     # Centralized cleaning
     df <- clean_dwd_data(df)
 
+    # Check availability
     has_global <- "solar_global" %in% names(df) && any(!is.na(df$solar_global))
     has_sun <- "sunshine_duration" %in% names(df) && any(!is.na(df$sunshine_duration))
 
     if (!has_global && !has_sun) {
-        return(create_empty_plot("Solar radiation and sunshine data not available"))
+        # It's possible the data exists in the file but not this window.
+        return(create_empty_plot("No solar/sunshine data in selected time range"))
     }
 
+    # --- Daily Aggregation ---
+    df_daily <- df %>%
+        dplyr::mutate(date = as.Date(datetime)) %>%
+        dplyr::group_by(date) %>%
+        dplyr::summarise(
+            # Sum J/cm^2 per hour -> Total Daily Energy J/cm^2
+            daily_solar = if (all(is.na(solar_global))) NA_real_ else sum(solar_global, na.rm = TRUE),
+
+            # Sum Minutes -> Convert to Hours
+            daily_sun_hours = if (all(is.na(sunshine_duration))) NA_real_ else sum(sunshine_duration, na.rm = TRUE) / 60,
+            .groups = "drop"
+        )
+
     date_range_str <- paste(
-        format(min(df$datetime), "%d %b %Y"), "-",
-        format(max(df$datetime), "%d %b %Y")
+        format(min(df_daily$date), "%d %b %Y"), "-",
+        format(max(df_daily$date), "%d %b %Y")
     )
 
-    p <- plotly::plot_ly(type = "scatter", mode = "none")
-
-    # Global Radiation
-    if ("solar_global" %in% names(df)) {
-        p <- p %>% plotly::add_lines(
-            data = df %>% dplyr::filter(!is.na(solar_global)),
-            x = ~datetime,
-            y = ~solar_global, name = "Global Rad (J/cm²)",
-            line = list(color = "#ffb300"), type = "scatter", mode = "lines",
-            showlegend = TRUE
-        )
+    # Base Plot
+    p <- plotly::plot_ly()
+
+    # 1. Global Radiation (Filled Area, Primary Y-Axis)
+    # Intuition: "Total Energy per Day"
+    if (has_global) {
+        sub_g <- df_daily %>% dplyr::filter(!is.na(daily_solar))
+        if (nrow(sub_g) > 0) {
+            p <- p %>% plotly::add_trace(
+                data = sub_g,
+                x = ~date,
+                y = ~daily_solar,
+                name = "Daily Global Radiation",
+                type = "scatter",
+                mode = "lines",
+                fill = "tozeroy",
+                line = list(color = "rgba(255, 179, 0, 0.9)", width = 1), # Amber
+                fillcolor = "rgba(255, 179, 0, 0.3)",
+                hovertemplate = "Daily Rad: %{y:,.0f} J/cm²<extra></extra>", # Comma for thousands
+                yaxis = "y"
+            )
+        }
     }
 
-    # Sunshine
-    if ("sunshine_duration" %in% names(df)) {
-        p <- p %>% plotly::add_lines(
-            data = df %>% dplyr::filter(!is.na(sunshine_duration)),
-            x = ~datetime,
-            y = ~sunshine_duration, name = "Sunshine (min)",
-            line = list(color = "#fbc02d", dash = "dot"), type = "scatter", mode = "lines",
-            showlegend = TRUE
-        )
+    # 2. Sunshine Duration (Bars, Secondary Y-Axis)
+    # Intuition: "Total Hours per Day"
+    if (has_sun) {
+        sub_s <- df_daily %>% dplyr::filter(!is.na(daily_sun_hours))
+        if (nrow(sub_s) > 0) {
+            # We use bar chart for sunshine
+            p <- p %>% plotly::add_bars(
+                data = sub_s,
+                x = ~date,
+                y = ~daily_sun_hours,
+                name = "Daily Sunshine",
+                marker = list(color = "rgba(0, 0, 0, 0.3)", line = list(color = "#FFD700", width = 1.5)), # Gold/Yellow outline
+                hovertemplate = "Sunshine: %{y:.1f} h<extra></extra>",
+                yaxis = "y2"
+            )
+        }
     }
 
+    # Layout Configuration for Dual Axis
     p %>%
         plotly::layout(
-            title = list(text = paste("Solar Radiation & Sunshine:", date_range_str), font = list(size = 14)),
-            xaxis = list(title = "", type = "date"),
-            yaxis = list(title = "Value"),
+            title = list(text = paste("Daily Solar Energy & Sunshine:", date_range_str), font = list(size = 14)),
+            xaxis = list(
+                title = "",
+                type = "date",
+                domain = c(0.05, 0.95)
+            ),
+            yaxis = list(
+                title = "Daily Radiation (J/cm²)",
+                titlefont = list(color = "#ffb300"),
+                tickfont = list(color = "#ffb300"),
+                side = "left",
+                showgrid = TRUE
+            ),
+            yaxis2 = list(
+                title = "Sunshine (Hours)",
+                titlefont = list(color = "#FFD700"),
+                tickfont = list(color = "#FFD700"),
+                side = "right",
+                overlaying = "y",
+                range = c(0, 18), # Max theoretical sunshine is < 17h in Germany
+                showgrid = FALSE
+            ),
             hovermode = "x unified",
-            hoverlabel = list()
+            hoverlabel = list(),
+            legend = list(orientation = "h", x = 0.5, xanchor = "center", y = 1.1),
+            margin = list(t = 60, b = 40, l = 60, r = 60),
+            modebar = list(orientation = "h")
         ) %>%
         plotly::config(displaylogo = FALSE)
 }

global.R

@@ -42,13 +42,23 @@ if (!dir.exists("data")) dir.create("data")
 index_file <- "data/dwd_file_index.rds"
 station_file <- "data/dwd_stations.rds"
 
-# 1. File Index
-if (!file.exists(index_file)) {
-    message("Building DWD File Index (this may take 10-20 seconds)...")
+# 1. File Index (with 24h Expiry)
+if (file.exists(index_file)) {
+    # Check age
+    info <- file.info(index_file)
+    age_hours <- difftime(Sys.time(), info$mtime, units = "hours")
+
+    if (age_hours > 24) {
+        message(paste("File index is", round(age_hours, 1), "hours old. Rebuilding..."))
+        dwd_index <- fetch_dwd_file_index()
+        saveRDS(dwd_index, index_file)
+    } else {
+        dwd_index <- readRDS(index_file)
+    }
+} else {
+    message("Building DWD File Index (first run)...")
     dwd_index <- fetch_dwd_file_index()
     saveRDS(dwd_index, index_file)
-} else {
-    dwd_index <- readRDS(index_file)
 }
 
 # 2. Station List
@@ -60,6 +70,56 @@ if (!file.exists(station_file)) {
     stations <- readRDS(station_file)
 }
 
+# 3. Enhance Station Metadata with Detailed Availability
+# Group DWD Index by ID to get ranges per parameter
+if (!is.null(dwd_index) && nrow(stations) > 0) {
+    param_summary <- dwd_index %>%
+        group_by(id, param) %>%
+        summarise(
+            # Combine ranges
+            # Suppress warnings for Inf/NA when no hist data exists
+            min_start = suppressWarnings(min(start_date, na.rm = TRUE)),
+            max_end = suppressWarnings(max(end_date, na.rm = TRUE)),
+            has_recent = any(type == "recent"),
+            has_hist = any(type == "historical"),
+            .groups = "drop"
+        ) %>%
+        mutate(
+            # Logic:
+            # If has_hist, we have a start date (e.g. 19800101) as character
+            # If has_recent only, min_start is Inf (numeric)
+
+            # Start Year
+            s_year = ifelse(is.character(min_start), substr(min_start, 1, 4), "Recent"),
+
+            # End Year
+            # If has_recent, assume "Present"
+            # If only hist, uses max_end (character)
+            e_year = ifelse(has_recent, "Present",
+                ifelse(is.character(max_end), substr(max_end, 1, 4), "Unknown")
+            ),
+
+            # Formatting
+            # Avoid "Recent-Present", just say "Recent"
+            label = ifelse(s_year == "Recent" & e_year == "Present",
+                paste0(str_to_title(param), " (Recent)"),
+                paste0(str_to_title(param), " (", s_year, "-", e_year, ")")
+            )
+        ) %>%
+        group_by(id) %>%
+        summarise(
+            detailed_summary = paste(label, collapse = ", ")
+        )
+
+    # Join back to stations
+    stations <- stations %>%
+        left_join(param_summary, by = "id") %>%
+        mutate(
+            # Fallback
+            detailed_summary = ifelse(is.na(detailed_summary), "No Data", detailed_summary)
+        )
+}
+
 # 3. Enrich Station List with Available Parameters
 if (exists("dwd_index") && exists("stations")) {
     param_summary <- dwd_index %>%

server.R

@@ -8,27 +8,75 @@ server <- function(input, output, session) {
 
     # Filtered stations based on Sidebar inputs
     filtered_stations <- reactive({
-        req(stations)
+        req(stations, input$date_range)
         df <- stations
 
-        # Filter by State
-        if (!is.null(input$state_filter)) {
-            df <- df %>% filter(state %in% input$state_filter)
-        }
-
         # Filter by Date (metadata availability)
-        # Using 'recent' file implies they are active or recent.
-        # We don't have start/end year easily compatible with date range without more parsing.
-        # For now, we assume all stations in the list are "available".
+        # We check if the station was active during any part of the selected range
+        range_start <- as.numeric(format(input$date_range[1], "%Y%m%d"))
+        range_end <- as.numeric(format(input$date_range[2], "%Y%m%d"))
+
+        df <- df %>% filter(
+            as.numeric(start_date) <= range_end &
+                as.numeric(end_date) >= range_start
+        )
 
         df
     })
 
-    # Initialize State Choices
+    # Initialize / Update Station Selector Choices
     observe({
-        req(stations)
-        states <- sort(unique(stations$state))
-        updateSelectInput(session, "state_filter", choices = states)
+        df <- filtered_stations()
+        req(df)
+
+        # Create choices: "Station Name (ID)" = "ID"
+        choices <- setNames(df$id, paste0(df$name, " (", df$id, ")"))
+
+        # Preserve selection if still in filtered list
+        current_sel <- input$station_selector
+
+        updateSelectizeInput(session, "station_selector",
+            choices = choices,
+            selected = current_sel,
+            server = TRUE
+        )
+    })
+
+    # --- Selection Logic via Dropdown ---
+    observeEvent(input$station_selector, {
+        req(input$station_selector)
+        id_val <- input$station_selector
+
+        # Find station details
+        s_meta <- stations %>% filter(id == id_val)
+        req(nrow(s_meta) > 0)
+
+        lat_val <- s_meta$latitude[1]
+        lng_val <- s_meta$longitude[1]
+
+        # Set ID
+        current_station_id(id_val)
+
+        # Highlight & Zoom
+        leafletProxy("map") %>%
+            setView(lng = lng_val + 0.8, lat = lat_val, zoom = 9) %>%
+            removeMarker(layerId = "selected-highlight") %>%
+            addCircleMarkers(
+                lng = lng_val, lat = lat_val,
+                layerId = "selected-highlight",
+                radius = 8, color = "red", fill = FALSE, opacity = 1, weight = 3
+            )
+
+        # Trigger Panel
+        output$show_details_panel <- reactive(TRUE)
+        outputOptions(output, "show_details_panel", suspendWhenHidden = FALSE)
+
+        # Update Dates to last 366 days of available data for this station
+        e_date <- tryCatch(as.Date(as.character(s_meta$end_date[1]), format = "%Y%m%d"), error = function(e) Sys.Date())
+        if (is.na(e_date)) e_date <- Sys.Date()
+        s_date <- e_date - 366
+        updateDateInput(session, "modal_date_start", value = s_date)
+        updateDateInput(session, "modal_date_end", value = e_date)
     })
 
     # Station Count
@@ -83,8 +131,8 @@ server <- function(input, output, session) {
         # Pre-process label with HTML
         df <- df %>% mutate(
             label_html = purrr::pmap_chr(
-                list(name, id, state, start_date, end_date, available_params),
-                function(name, id, state, start_date, end_date, available_params) {
+                list(name, id, state, start_date, end_date, detailed_summary),
+                function(name, id, state, start_date, end_date, detailed_summary) {
                     tryCatch(
                         {
                             s <- as.Date(as.character(start_date), format = "%Y%m%d")
@@ -95,11 +143,11 @@ server <- function(input, output, session) {
                                 "<b>", htmltools::htmlEscape(name), "</b> (", id, ")<br>",
                                 "<span style='font-size:80%;'>State: ", htmltools::htmlEscape(state), "</span><br>",
                                 "<span style='font-size:80%; color:#555;'>",
-                                "Period: ", format(s, "%Y-%m-%d"), " &ndash; ", format(e, "%Y-%m-%d"),
+                                "Station Active: ", format(s, "%Y-%m-%d"), " &ndash; ", format(e, "%Y-%m-%d"),
                                 "</span><br>",
-                                "<div style='font-size:80%; color:#333; font-weight:bold; margin-top:5px;'>Available Parameters:</div>",
-                                "<div style='font-size:75%; color:#333; font-style:italic; line-height: 1.1;'>• ",
-                                gsub(", ", "<br>• ", htmltools::htmlEscape(available_params)),
+                                "<div style='font-size:80%; color:#333; font-weight:bold; margin-top:5px;'>Data Availability:</div>",
+                                "<div style='font-size:75%; color:#333; line-height: 1.1;'>• ",
+                                gsub(", ", "<br>• ", htmltools::htmlEscape(detailed_summary)),
                                 "</div>",
                                 "</div>"
                             )
@@ -131,6 +179,19 @@ server <- function(input, output, session) {
         # Set ID
         current_station_id(id_val)
 
+        # Highlight & Zoom
+        leafletProxy("map") %>%
+            setView(lng = click$lng + 0.8, lat = click$lat, zoom = 9) %>%
+            removeMarker(layerId = "selected-highlight") %>%
+            addCircleMarkers(
+                lng = click$lng, lat = click$lat,
+                layerId = "selected-highlight",
+                radius = 8, color = "red", fill = FALSE, opacity = 1, weight = 3
+            )
+
+        # Sync dropdown
+        updateSelectizeInput(session, "station_selector", selected = id_val)
+
         # Trigger Panel
         output$show_details_panel <- reactive(TRUE)
         outputOptions(output, "show_details_panel", suspendWhenHidden = FALSE)
@@ -503,12 +564,43 @@ server <- function(input, output, session) {
                     distinct(datetime, .keep_all = TRUE) %>%
                     arrange(datetime)
 
-                # Final Window Filter (safety)
+                # Final Window Filter (safety) with Auto-Correction
                 s_date <- window_debounced()$start
                 e_date <- window_debounced()$end
+
                 if (!is.null(s_date) && !is.null(e_date)) {
-                    clean_df <- clean_df %>%
-                        filter(datetime >= as.POSIXct(s_date), datetime <= as.POSIXct(e_date) + days(1))
+                    req_start <- as.POSIXct(s_date)
+                    req_end <- as.POSIXct(e_date) + days(1)
+
+                    filtered_df <- clean_df %>%
+                        filter(datetime >= req_start, datetime <= req_end)
+
+                    # Check for mismatch: Data exists but not in window
+                    if (nrow(filtered_df) == 0 && nrow(clean_df) > 0) {
+                        data_max <- max(clean_df$datetime, na.rm = TRUE)
+                        data_min <- min(clean_df$datetime, na.rm = TRUE)
+
+                        # If the requested window is completely outside the data range
+                        if (req_start > data_max || req_end < data_min) {
+                            msg <- paste0("No data in selected range. Auto-adjusting to latest available: ", format(data_max, "%Y-%m-%d"))
+                            loading_diagnostics(msg)
+
+                            # Auto-adjust dates
+                            # This will trigger a re-fetch via the observer
+                            new_end <- as.Date(data_max)
+                            new_start <- new_end - 366
+
+                            updateDateInput(session, "modal_date_start", value = new_start)
+                            # Small delay for end date to ensure sliding window logic doesn't fight us
+                            # confusing interaction possible, but setting both usually works
+                            updateDateInput(session, "modal_date_end", value = new_end)
+
+                            # Stop this cycle
+                            return()
+                        }
+                    }
+
+                    clean_df <- filtered_df
                 }
 
                 station_data(clean_df)

ui.R

@@ -86,8 +86,8 @@ ui <- page_navbar(
     theme = bs_theme(version = 5, bootswatch = "cosmo"),
     sidebar = sidebar(
         title = "Filters",
-        # State Filter
-        selectInput("state_filter", "Bundesland (State)", choices = NULL, multiple = TRUE),
+        # Station Search
+        selectizeInput("station_selector", "Find Station", choices = NULL, multiple = FALSE, options = list(placeholder = "Type to search...")),
         div(
             style = "display: flex; align-items: center; gap: 8px;",
             tags$label("Select Date Range"),