Update app.py

app.py

@@ -119,15 +119,36 @@ VALID_SEVERITY   = {"low", "medium", "high"}
 VALID_CONFIDENCE = {"low", "medium", "high"}
 VALID_PRIORITY   = {"low", "medium", "high"}
 
+
 WEATHER_VARIABLES = [
-    "temperature_2m_max", "temperature_2m_min", "rain_sum", "precipitation_sum",
-    "wind_gusts_10m_max", "shortwave_radiation_sum", "temperature_2m_mean",
-    "cloud_cover_mean", "et0_fao_evapotranspiration",
-    "growing_degree_days_base_0_limit_50", "leaf_wetness_probability_mean",
-    "vapour_pressure_deficit_max"
+    # Temperature and plant development
+    "temperature_2m_max",
+    "temperature_2m_min",
+    "temperature_2m_mean",
+    "growing_degree_days_base_0_limit_50",
+
+    # Rainfall and water balance
+    "precipitation_sum",
+    "precipitation_hours",
+    "precipitation_probability_max",
+    "et0_fao_evapotranspiration",
+
+    # Humidity and disease-related conditions
+    "relative_humidity_2m_mean",
+    "relative_humidity_2m_max",
+    "dew_point_2m_mean",
+    "leaf_wetness_probability_mean",
+    "vapour_pressure_deficit_max",
+
+    # Radiation, clouds, and wind
+    "shortwave_radiation_sum",
+    "cloud_cover_mean",
+    "wind_speed_10m_max",
+    "wind_gusts_10m_max",
 ]
 
 
+
 # ------------------------------------------------------------------
 # DATA CLASSES
 # ------------------------------------------------------------------
@@ -780,6 +801,7 @@ def handle_weather_and_analyze(weather_json_str: str, lat: float, lon: float, cr
             "current_concern": current_concern,
         },
         "raw_weather": raw_weather,
+        "derived_features": normalized.get("derived_features", {}),
         "analysis": llm_output,
         "created_at": datetime.now(timezone.utc).isoformat()
     }
@@ -829,6 +851,480 @@ def handle_weather_and_analyze(weather_json_str: str, lat: float, lon: float, cr
 # ------------------------------------------------------------------
 # 4. ANALYTICS ENGINE
 # ------------------------------------------------------------------
+
+
+
+# ------------------------------------------------------------------
+# AGRICULTURAL WEATHER FEATURE ENGINEERING
+# ------------------------------------------------------------------
+
+def _weather_num(value):
+    """Convert valid finite numeric values to float; otherwise None."""
+    try:
+        if value is None:
+            return None
+        number = float(value)
+        return number if math.isfinite(number) else None
+    except (TypeError, ValueError):
+        return None
+
+
+def _weather_probability_fraction(value, unit=None):
+    """
+    Normalize a probability to 0-1.
+    Open-Meteo probability variables commonly use percent units.
+    """
+    number = _weather_num(value)
+    if number is None:
+        return None
+
+    if unit and "%" in str(unit):
+        number /= 100.0
+    elif number > 1.0:
+        number /= 100.0
+
+    return max(0.0, min(1.0, number))
+
+
+def engineer_weather_features(normalized):
+    """
+    Build derived agricultural weather features from the existing
+    daily arrays. No crop, soil, or sensor input is required.
+
+    Input:
+        normalized["historical"], normalized["today"],
+        normalized["forecast"], normalized["units"]
+
+    Output:
+        Separate daily feature arrays plus period summaries.
+    """
+
+    units = normalized.get("units", {}) or {}
+    periods = ("historical", "today", "forecast")
+    records = []
+
+    # 1. Combine the three time periods chronologically.
+    for period in periods:
+        block = normalized.get(period, {}) or {}
+        dates = block.get("time", []) or []
+
+        for i, date in enumerate(dates):
+            raw = {}
+
+            for variable in WEATHER_VARIABLES:
+                values = block.get(variable, [])
+                value = (
+                    values[i]
+                    if isinstance(values, list) and i < len(values)
+                    else None
+                )
+                raw[variable] = _weather_num(value)
+
+            rh = raw.get("relative_humidity_2m_mean")
+            leaf_wetness = _weather_probability_fraction(
+                raw.get("leaf_wetness_probability_mean"),
+                units.get("leaf_wetness_probability_mean"),
+            )
+            precip_probability = _weather_probability_fraction(
+                raw.get("precipitation_probability_max"),
+                units.get("precipitation_probability_max"),
+            )
+
+            temperature_mean = raw.get("temperature_2m_mean")
+            temperature_min = raw.get("temperature_2m_min")
+            temperature_max = raw.get("temperature_2m_max")
+            dewpoint_mean = raw.get("dew_point_2m_mean")
+            precipitation = raw.get("precipitation_sum")
+            precipitation_hours = raw.get("precipitation_hours")
+            et0 = raw.get("et0_fao_evapotranspiration")
+
+            water_balance = (
+                precipitation - et0
+                if precipitation is not None and et0 is not None
+                else None
+            )
+
+            features = {
+                # Temperature
+                "temperature_range_c": (
+                    temperature_max - temperature_min
+                    if temperature_max is not None
+                    and temperature_min is not None
+                    else None
+                ),
+
+                "dewpoint_depression_proxy_c": (
+                    temperature_mean - dewpoint_mean
+                    if temperature_mean is not None
+                    and dewpoint_mean is not None
+                    else None
+                ),
+
+                # Water balance proxy
+                "water_balance_proxy_mm": water_balance,
+
+                "precipitation_per_wet_hour_proxy": (
+                    precipitation / precipitation_hours
+                    if precipitation is not None
+                    and precipitation_hours is not None
+                    and precipitation_hours > 0
+                    else None
+                ),
+
+                # Normalized probabilities
+                "leaf_wetness_fraction": leaf_wetness,
+                "precip_probability_fraction": precip_probability,
+
+                # Humidity and leaf-wetness interactions.
+                # These are environmental proxies, NOT disease probabilities.
+                "humidity_wetness_interaction": (
+                    (rh / 100.0) * leaf_wetness
+                    if rh is not None and leaf_wetness is not None
+                    else None
+                ),
+
+                "wetness_duration_interaction": (
+                    leaf_wetness
+                    * max(0.0, min(1.0, precipitation_hours / 24.0))
+                    if leaf_wetness is not None
+                    and precipitation_hours is not None
+                    else None
+                ),
+            }
+
+            records.append({
+                "date": date,
+                "period": period,
+                "raw": raw,
+                "features": features,
+            })
+
+    # 2. Consecutive days with nearly zero recorded precipitation.
+    # 0.1 mm is an operational definition of a nearly rain-free day,
+    # not a universal crop drought threshold.
+    dry_streak = 0
+
+    for record in records:
+        precipitation = record["raw"].get("precipitation_sum")
+
+        if precipitation is None:
+            dry_streak = 0
+            record["features"]["dry_spell_days"] = None
+        elif precipitation <= 0.1:
+            dry_streak += 1
+            record["features"]["dry_spell_days"] = dry_streak
+        else:
+            dry_streak = 0
+            record["features"]["dry_spell_days"] = 0
+
+    # 3. Trailing rolling sums and means across the continuous timeline.
+    # Missing values are not replaced with zero.
+    def rolling_value(index, source, key, window, operation):
+        if index + 1 < window:
+            return None
+
+        subset = records[index - window + 1:index + 1]
+
+        if source == "raw":
+            values = [r["raw"].get(key) for r in subset]
+        else:
+            values = [r["features"].get(key) for r in subset]
+
+        if any(v is None for v in values):
+            return None
+
+        if operation == "sum":
+            return sum(values)
+
+        return sum(values) / len(values)
+
+    rolling_specs = {
+        "precipitation_sum": ("raw", "precipitation_sum", "sum", "precipitation"),
+        "et0_fao_evapotranspiration": (
+            "raw", "et0_fao_evapotranspiration", "sum", "et0"
+        ),
+        "water_balance_proxy_mm": (
+            "feature", "water_balance_proxy_mm", "sum", "water_balance"
+        ),
+        "growing_degree_days_base_0_limit_50": (
+            "raw", "growing_degree_days_base_0_limit_50", "sum", "gdd"
+        ),
+        "temperature_2m_mean": (
+            "raw", "temperature_2m_mean", "mean", "temperature_mean"
+        ),
+        "vapour_pressure_deficit_max": (
+            "raw", "vapour_pressure_deficit_max", "mean", "vpd"
+        ),
+        "relative_humidity_2m_mean": (
+            "raw", "relative_humidity_2m_mean", "mean", "rh_mean_pct"
+        ),
+        "leaf_wetness_fraction": (
+            "feature", "leaf_wetness_fraction", "mean", "leaf_wetness"
+        ),
+    }
+
+    for i, record in enumerate(records):
+        for window in (3, 7):
+            for _, (source, key, operation, prefix) in rolling_specs.items():
+                result_key = f"{prefix}_{window}d_{operation}"
+                record["features"][result_key] = rolling_value(
+                    i, source, key, window, operation
+                )
+
+    # 4. One-day changes; values compare against the immediately
+    # preceding calendar day, including across history/today/forecast.
+    delta_specs = {
+        "temperature_mean_delta_1d": (
+            "raw", "temperature_2m_mean"
+        ),
+        "vpd_delta_1d": (
+            "raw", "vapour_pressure_deficit_max"
+        ),
+        "rh_mean_delta_1d": (
+            "raw", "relative_humidity_2m_mean"
+        ),
+        "precipitation_delta_1d": (
+            "raw", "precipitation_sum"
+        ),
+        "leaf_wetness_delta_1d": (
+            "feature", "leaf_wetness_fraction"
+        ),
+    }
+
+    for i, record in enumerate(records):
+        for feature_name in delta_specs:
+            record["features"][feature_name] = None
+
+        if i == 0:
+            continue
+
+        previous = records[i - 1]
+
+        for feature_name, (source, key) in delta_specs.items():
+            current_value = (
+                record["raw"].get(key)
+                if source == "raw"
+                else record["features"].get(key)
+            )
+            previous_value = (
+                previous["raw"].get(key)
+                if source == "raw"
+                else previous["features"].get(key)
+            )
+
+            if current_value is not None and previous_value is not None:
+                record["features"][feature_name] = (
+                    current_value - previous_value
+                )
+
+    # 5. Forecast daily values relative to the historical 7-day mean.
+    anomaly_specs = {
+        "temperature_mean_delta_vs_history7d": (
+            "raw", "temperature_2m_mean"
+        ),
+        "vpd_delta_vs_history7d": (
+            "raw", "vapour_pressure_deficit_max"
+        ),
+        "rh_mean_delta_vs_history7d": (
+            "raw", "relative_humidity_2m_mean"
+        ),
+        "precipitation_delta_vs_history7d": (
+            "raw", "precipitation_sum"
+        ),
+        "et0_delta_vs_history7d": (
+            "raw", "et0_fao_evapotranspiration"
+        ),
+        "leaf_wetness_delta_vs_history7d": (
+            "feature", "leaf_wetness_fraction"
+        ),
+    }
+
+    historical_records = [
+        r for r in records if r["period"] == "historical"
+    ]
+
+    baselines = {}
+
+    for feature_name, (source, key) in anomaly_specs.items():
+        values = [
+            r["raw"].get(key) if source == "raw"
+            else r["features"].get(key)
+            for r in historical_records
+        ]
+        values = [v for v in values if v is not None]
+
+        baselines[feature_name] = (
+            sum(values) / len(values) if values else None
+        )
+
+    for record in records:
+        for feature_name, (source, key) in anomaly_specs.items():
+            record["features"][feature_name] = None
+
+            if record["period"] != "forecast":
+                continue
+
+            value = (
+                record["raw"].get(key) if source == "raw"
+                else record["features"].get(key)
+            )
+            baseline = baselines[feature_name]
+
+            if value is not None and baseline is not None:
+                record["features"][feature_name] = value - baseline
+
+    # 6. Return feature arrays grouped by the original time windows.
+    all_feature_names = sorted({
+        key
+        for record in records
+        for key in record["features"]
+    })
+
+    derived_blocks = {}
+
+    for period in periods:
+        period_records = [
+            r for r in records if r["period"] == period
+        ]
+
+        derived_blocks[period] = {
+            "time": [r["date"] for r in period_records],
+            "features": {
+                name: [
+                    r["features"].get(name)
+                    for r in period_records
+                ]
+                for name in all_feature_names
+            },
+        }
+
+    # 7. Compact period-level summary for the LLM.
+    def mean_values(values):
+        values = [v for v in values if v is not None]
+        return sum(values) / len(values) if values else None
+
+    def total_values(values):
+        # If any daily value is missing, avoid presenting a partial
+        # sum as the complete period total.
+        if not values or any(v is None for v in values):
+            return None
+        return sum(values)
+
+    def summarize_period(period):
+        subset = [r for r in records if r["period"] == period]
+
+        def raw_values(key):
+            return [r["raw"].get(key) for r in subset]
+
+        def feature_values(key):
+            return [r["features"].get(key) for r in subset]
+
+        return {
+            "days": len(subset),
+            "temperature_mean_c": mean_values(
+                raw_values("temperature_2m_mean")
+            ),
+            "temperature_max_peak_c": (
+                max([v for v in raw_values("temperature_2m_max")
+                     if v is not None], default=None)
+            ),
+            "temperature_min_low_c": (
+                min([v for v in raw_values("temperature_2m_min")
+                     if v is not None], default=None)
+            ),
+            "temperature_range_mean_c": mean_values(
+                feature_values("temperature_range_c")
+            ),
+            "precipitation_total_mm": total_values(
+                raw_values("precipitation_sum")
+            ),
+            "precipitation_hours_total": total_values(
+                raw_values("precipitation_hours")
+            ),
+            "et0_total_mm": total_values(
+                raw_values("et0_fao_evapotranspiration")
+            ),
+            "water_balance_proxy_total_mm": total_values(
+                feature_values("water_balance_proxy_mm")
+            ),
+            "gdd_total": total_values(
+                raw_values("growing_degree_days_base_0_limit_50")
+            ),
+            "vpd_mean": mean_values(
+                raw_values("vapour_pressure_deficit_max")
+            ),
+            "relative_humidity_mean_pct": mean_values(
+                raw_values("relative_humidity_2m_mean")
+            ),
+            "leaf_wetness_mean_fraction": mean_values(
+                feature_values("leaf_wetness_fraction")
+            ),
+            "humidity_wetness_interaction_mean": mean_values(
+                feature_values("humidity_wetness_interaction")
+            ),
+            "longest_dry_spell_days": max(
+                [v for v in feature_values("dry_spell_days")
+                 if v is not None],
+                default=None
+            ),
+            "max_wind_speed": max(
+                [v for v in raw_values("wind_speed_10m_max")
+                 if v is not None],
+                default=None
+            ),
+            "max_wind_gust": max(
+                [v for v in raw_values("wind_gusts_10m_max")
+                 if v is not None],
+                default=None
+            ),
+            "precip_probability_mean_fraction": mean_values(
+                feature_values("precip_probability_fraction")
+            ),
+        }
+
+    history_summary = summarize_period("historical")
+    forecast_summary = summarize_period("forecast")
+
+    comparison_keys = [
+        "temperature_mean_c",
+        "precipitation_total_mm",
+        "et0_total_mm",
+        "water_balance_proxy_total_mm",
+        "gdd_total",
+        "vpd_mean",
+        "relative_humidity_mean_pct",
+        "leaf_wetness_mean_fraction",
+    ]
+
+    comparison = {}
+
+    for key in comparison_keys:
+        h = history_summary.get(key)
+        f = forecast_summary.get(key)
+
+        comparison[key + "_forecast_minus_history"] = (
+            f - h if h is not None and f is not None else None
+        )
+
+    return {
+        "historical": derived_blocks["historical"],
+        "today": derived_blocks["today"],
+        "forecast": derived_blocks["forecast"],
+        "summary": {
+            "history_7d": history_summary,
+            "forecast_7d": forecast_summary,
+            "forecast_minus_history": comparison,
+        },
+    }
+
+
+
+
+
+
+
+
+
 def normalize_weather_data(raw_data: Dict, lat: float, lon: float, client_date: str) -> Tuple[Optional[Dict], str]:
     try:
         timezone_str = raw_data.get("timezone", "UTC")
@@ -880,6 +1376,12 @@ def normalize_weather_data(raw_data: Dict, lat: float, lon: float, client_date:
             "today": today,
             "forecast": forecast
         }
+
+
+        normalized["derived_features"] = engineer_weather_features(
+            normalized
+        )
+
         return normalized, ""
 
     except Exception as e:
@@ -914,6 +1416,49 @@ def build_llm_payload(normalized_data: Dict, lat: float, lon: float, client_date
             "location_timezone": location.get("timezone", "UTC")
         },
         "units": normalized_data.get("units", {}),
+
+
+
+        # NEW: engineered agricultural weather features
+        "derived_features": normalized_data.get(
+            "derived_features", {}
+        ),
+
+        "derived_feature_guide": {
+            "temperature_range_c":
+                "Daily maximum temperature minus minimum temperature.",
+            "dewpoint_depression_proxy_c":
+                "Mean air temperature minus mean dewpoint.",
+            "water_balance_proxy_mm":
+                "Precipitation minus reference ET0; not soil water balance.",
+            "precipitation_per_wet_hour_proxy":
+                "Daily precipitation divided by precipitation hours; "
+                "not peak rainfall intensity.",
+            "humidity_wetness_interaction":
+                "Relative humidity fraction multiplied by leaf-wetness "
+                "probability fraction; not disease probability.",
+            "wetness_duration_interaction":
+                "Leaf-wetness probability multiplied by precipitation "
+                "hours divided by 24; not measured leaf-wetness duration.",
+            "dry_spell_days":
+                "Consecutive days with precipitation at or below "
+                "0.1 mm; not a crop-specific drought threshold.",
+            "rolling_features":
+                "Trailing 3-day and 7-day sums or means. Windows span "
+                "the continuous history, today, and forecast timeline.",
+            "delta_vs_history7d":
+                "Forecast daily value minus the mean of available "
+                "historical daily values over the previous 7 days.",
+            "forecast_minus_history":
+                "Difference between the 7-day forecast summary and "
+                "the 7-day historical summary.",
+            "probability_fraction":
+                "Normalized to a 0-1 scale when the source unit is "
+                "percent. Null means unavailable, not zero."
+        },
+
+
+
         "historical": {"daily": normalized_data.get("historical", {})},
         "forecast": {"daily": normalized_data.get("forecast", {})}
     }
@@ -1026,12 +1571,38 @@ def build_export_file(cache_entry: Optional[Dict]) -> Optional[str]:
     lines.append(f"- Current Concern   : {field_context.get('current_concern') or '-'}")
     lines.append("")
     lines.append("--- DATA CUACA MENTAH (Ecwf, historical 7 hari + forecast 7 hari) ---")
-    lines.append(json.dumps(cache_entry.get("raw_weather", {}), ensure_ascii=False, indent=2))
+
+
+    lines.append("--- DATA CUACA MENTAH (Open-Meteo, historical + forecast) ---")
+    lines.append(json.dumps(
+        cache_entry.get("raw_weather", {}),
+        ensure_ascii=False,
+        indent=2
+    ))
     lines.append("")
+
+    # FEATURE ENGINEERING: tambahkan tepat di sini
+    lines.append("--- FITUR CUACA TURUNAN ---")
+    lines.append(
+        json.dumps(
+            cache_entry.get("derived_features", {}),
+            ensure_ascii=False,
+            indent=2,
+            allow_nan=False
+        )
+    )
+    lines.append("")
+
+    # Bagian analisis Gemini tetap dilanjutkan seperti sebelumnya
     lines.append("--- HASIL ANALISIS GEMINI AI ---")
     lines.append(f"Ringkasan: {analysis.get('summary', '-')}")
     lines.append("")
 
+
+
+    lines.append(f"Ringkasan: {analysis.get('summary', '-')}")
+    lines.append("")
+
     hist = analysis.get("historical", {}) or {}
     lines.append("Historical (7 hari terakhir):")
     lines.append(f"  Kondisi : {hist.get('condition', '-')}")
@@ -1455,6 +2026,7 @@ def handle_analyze(lat: float, lon: float, state_dict: Dict):
             "current_concern": current_concern,
         },
         "raw_weather": raw_weather,
+        "derived_features": normalized.get("derived_features", {}),
         "analysis": llm_output,
         "created_at": datetime.now(timezone.utc).isoformat()
     }