v1.0.0 TimeSeries Service with AutoNHITS

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.10
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libgdal-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first for caching
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application files
+COPY app.py .
+COPY satellite_pipeline.py .
+COPY auto_tuning_predictor.py .
+COPY prediction_pipeline.py .
+
+# Expose port
+EXPOSE 7860
+
+# Run the application
+CMD ["python", "app.py"]

README.md

@@ -0,0 +1,57 @@
+---
+title: AGROW TimeSeries
+emoji: 📈
+colorFrom: green
+colorTo: blue
+sdk: docker
+pinned: false
+license: mit
+---
+
+# AGROW Time Series Service
+
+Time series forecasting for satellite indices using AutoNHITS.
+
+## Features
+
+- Historical data from Sentinel-1 (SAR) and Sentinel-2 (Optical)
+- AutoNHITS forecasting with weather integration
+- Returns JSON data for interactive Flutter charts
+
+## API Endpoints
+
+### POST /timeseries
+
+Generate time series with historical data and forecasts.
+
+**Request:**
+```json
+{
+  "center_lat": 26.1885,
+  "center_lon": 91.6894,
+  "field_size_hectares": 10.0,
+  "metric": "VV"
+}
+```
+
+**Response:**
+```json
+{
+  "success": true,
+  "metric": "VV",
+  "historical": [{"date": "2024-01-01", "value": -12.5}, ...],
+  "forecast": [{"date": "2024-12-01", "value": -11.8, "confidence_low": -12.5, "confidence_high": -11.0}],
+  "trend": "improving",
+  "stats": {"min": -15.2, "max": -8.5, "mean": -11.5}
+}
+```
+
+## Supported Metrics
+
+- **SAR:** VV, VH (Sentinel-1)
+- **Optical:** B02-B12 (Sentinel-2)
+
+## Required Secrets
+
+- `SH_CLIENT_ID`
+- `SH_CLIENT_SECRET`

app.py

@@ -0,0 +1,276 @@
+"""
+AGROW Time Series Service
+=========================
+Time series forecasting for satellite indices using AutoNHITS.
+
+Features:
+- Historical data from Sentinel-1 (SAR) and Sentinel-2 (Optical)
+- AutoNHITS forecasting with weather integration
+- Returns JSON data points for interactive Flutter charts
+"""
+
+import os
+import io
+import json
+import base64
+import logging
+import traceback
+from datetime import datetime, timedelta
+from typing import List, Optional, Dict, Any
+
+import numpy as np
+import pandas as pd
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+
+# Import our modules
+from satellite_pipeline import SatelliteFetcher
+from auto_tuning_predictor import AutoTimeSeriesPredictor
+
+# ============================================================================
+# LOGGING
+# ============================================================================
+logging.basicConfig(
+    level=logging.INFO,
+    format='[%(asctime)s] %(levelname)s: %(message)s',
+    datefmt='%H:%M:%S'
+)
+logger = logging.getLogger("TimeSeriesService")
+
+# ============================================================================
+# FASTAPI
+# ============================================================================
+app = FastAPI(
+    title="AGROW Time Series Service",
+    description="Time series forecasting for satellite indices",
+    version="1.0.0"
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# ============================================================================
+# REQUEST/RESPONSE MODELS
+# ============================================================================
+class DataPoint(BaseModel):
+    date: str
+    value: float
+
+class ForecastPoint(BaseModel):
+    date: str
+    value: float
+    confidence_low: Optional[float] = None
+    confidence_high: Optional[float] = None
+
+class TimeSeriesRequest(BaseModel):
+    center_lat: float
+    center_lon: float
+    field_size_hectares: float = 10.0
+    metric: str = "VV"  # VV, VH, B04, B08, NDVI, etc.
+    days_history: int = 365  # How many days of history
+    days_forecast: int = 30   # How many days to forecast
+
+class TimeSeriesResponse(BaseModel):
+    success: bool
+    metric: str
+    historical: List[DataPoint]
+    forecast: List[ForecastPoint]
+    trend: str  # "improving", "stable", "declining"
+    stats: Dict[str, float]
+    timestamp: str
+
+# ============================================================================
+# HELPERS
+# ============================================================================
+def coords_to_polygon(center_lat: float, center_lon: float, size_ha: float):
+    """Convert center point and size to polygon coordinates."""
+    # Approximate conversion: 1 hectare = 0.01 km²
+    radius_km = np.sqrt(size_ha / 100) / 2
+    lat_off = radius_km / 111
+    lon_off = radius_km / (111 * np.cos(np.radians(center_lat)))
+    
+    return [
+        (center_lon - lon_off, center_lat - lat_off),
+        (center_lon + lon_off, center_lat - lat_off),
+        (center_lon + lon_off, center_lat + lat_off),
+        (center_lon - lon_off, center_lat + lat_off),
+        (center_lon - lon_off, center_lat - lat_off),  # Close polygon
+    ]
+
+def calculate_trend(values: List[float]) -> str:
+    """Determine trend from values."""
+    if len(values) < 5:
+        return "stable"
+    
+    # Simple linear regression slope
+    x = np.arange(len(values))
+    slope = np.polyfit(x, values, 1)[0]
+    
+    if slope > 0.01:
+        return "improving"
+    elif slope < -0.01:
+        return "declining"
+    return "stable"
+
+# ============================================================================
+# API ENDPOINTS
+# ============================================================================
+@app.get("/")
+async def root():
+    return {
+        "service": "AGROW Time Series Service",
+        "version": "1.0.0",
+        "metrics": ["VV", "VH", "B02", "B03", "B04", "B08", "B8A", "B11", "B12"],
+        "description": "Time series forecasting with AutoNHITS"
+    }
+
+@app.get("/health")
+async def health():
+    return {"status": "healthy"}
+
+
+@app.post("/timeseries", response_model=TimeSeriesResponse)
+async def get_timeseries(request: TimeSeriesRequest):
+    """
+    Fetch historical satellite data and generate forecasts.
+    
+    This uses the full AutoNHITS model for accurate predictions.
+    Note: First request may take 2-5 minutes for data fetching and model training.
+    """
+    req_id = datetime.now().strftime("%H%M%S")
+    logger.info(f"[{req_id}] Time Series Request: {request.metric}")
+    logger.info(f"[{req_id}] Location: ({request.center_lat}, {request.center_lon})")
+    
+    try:
+        # Step 1: Create polygon from center point
+        polygon = coords_to_polygon(
+            request.center_lat, 
+            request.center_lon, 
+            request.field_size_hectares
+        )
+        logger.info(f"[{req_id}] Polygon created: {polygon[:2]}...")
+        
+        # Step 2: Fetch historical satellite data
+        logger.info(f"[{req_id}] Fetching satellite data...")
+        fetcher = SatelliteFetcher(polygon)
+        
+        # Determine which data to fetch based on metric
+        if request.metric in ['VV', 'VH']:
+            fetcher.fetch_sar_data('sar_data.csv')
+            csv_file = 'sar_data.csv'
+            target_col = f'{request.metric}_mean_dB'
+        else:
+            fetcher.fetch_sentinel2_data('sentinel2_data.csv')
+            csv_file = 'sentinel2_data.csv'
+            target_col = request.metric
+        
+        # Read historical data
+        if not os.path.exists(csv_file):
+            raise HTTPException(404, "No satellite data available for this location")
+        
+        df = pd.read_csv(csv_file)
+        if 'ds' not in df.columns:
+            raise HTTPException(500, "Invalid data format")
+        
+        # Convert to DataPoint list
+        historical = []
+        for _, row in df.iterrows():
+            if target_col in row and not pd.isna(row[target_col]):
+                historical.append(DataPoint(
+                    date=str(row['ds']),
+                    value=round(float(row[target_col]), 4)
+                ))
+        
+        if len(historical) < 10:
+            raise HTTPException(400, f"Insufficient data points ({len(historical)}) for forecasting")
+        
+        logger.info(f"[{req_id}] Historical data: {len(historical)} points")
+        
+        # Step 3: Run AutoNHITS prediction
+        logger.info(f"[{req_id}] Running AutoNHITS prediction...")
+        predictor = AutoTimeSeriesPredictor()
+        
+        predictions = predictor.tune_and_predict(
+            csv_path=csv_file,
+            field_coords=polygon,
+            target_col=target_col,
+            output_file='predictions.csv',
+            num_samples=5  # Quick tuning for API
+        )
+        
+        # Convert predictions to ForecastPoint list
+        forecast = []
+        for _, row in predictions.iterrows():
+            value = float(row['predicted_y'])
+            forecast.append(ForecastPoint(
+                date=str(row['ds'].date()) if hasattr(row['ds'], 'date') else str(row['ds']),
+                value=round(value, 4),
+                confidence_low=round(value * 0.9, 4),  # Simple confidence band
+                confidence_high=round(value * 1.1, 4)
+            ))
+        
+        logger.info(f"[{req_id}] Forecast: {len(forecast)} points")
+        
+        # Calculate stats
+        all_values = [p.value for p in historical]
+        stats = {
+            "min": round(min(all_values), 4),
+            "max": round(max(all_values), 4),
+            "mean": round(sum(all_values) / len(all_values), 4),
+            "count": len(all_values),
+            "forecast_count": len(forecast)
+        }
+        
+        trend = calculate_trend(all_values[-20:] if len(all_values) > 20 else all_values)
+        
+        # Cleanup temp files
+        for f in ['sar_data.csv', 'sentinel2_data.csv', 'predictions.csv']:
+            if os.path.exists(f):
+                os.remove(f)
+        
+        logger.info(f"[{req_id}] SUCCESS - Trend: {trend}")
+        
+        return TimeSeriesResponse(
+            success=True,
+            metric=request.metric,
+            historical=historical,
+            forecast=forecast,
+            trend=trend,
+            stats=stats,
+            timestamp=datetime.now().isoformat()
+        )
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"[{req_id}] ERROR: {str(e)}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(500, str(e))
+
+
+@app.get("/timeseries/quick")
+async def quick_timeseries(
+    center_lat: float,
+    center_lon: float,
+    metric: str = "VV",
+    field_size_hectares: float = 10.0
+):
+    """Quick endpoint for simple GET requests."""
+    request = TimeSeriesRequest(
+        center_lat=center_lat,
+        center_lon=center_lon,
+        field_size_hectares=field_size_hectares,
+        metric=metric
+    )
+    return await get_timeseries(request)
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)

auto_tuning_predictor.py

@@ -0,0 +1,441 @@
+import pandas as pd
+import numpy as np
+from shapely.geometry import Polygon
+from sklearn.preprocessing import StandardScaler
+import matplotlib.pyplot as plt
+
+from neuralforecast import NeuralForecast
+from neuralforecast.auto import AutoNHITS
+from neuralforecast.losses.pytorch import MAE
+
+import openmeteo_requests
+import requests_cache
+from retry_requests import retry
+from openmeteo_sdk.Variable import Variable
+from openmeteo_sdk.Aggregation import Aggregation
+import warnings
+import os
+import datetime
+
+# Suppress warnings
+warnings.filterwarnings("ignore")
+
+class AutoTimeSeriesPredictor:
+    def __init__(self):
+        # Coordinates will be set in tune_and_predict
+        self.field_coords = None
+        self.latitude = None
+        self.longitude = None
+
+        # OpenMeteo Client Setup
+        cache_session = requests_cache.CachedSession(".cache", expire_after=-1)
+        retry_session = retry(cache_session, retries=5, backoff_factor=0.2)
+        self.openmeteo = openmeteo_requests.Client(session=retry_session)
+
+        # Feature Configuration
+        self.hist_exog_list = ["lag1", "diff"]
+        self.futr_exog_list = ["temp", "rainfall", "humidity"]
+
+        # Scalers
+        self.y_scaler = StandardScaler()
+        self.exog_scaler = StandardScaler()
+
+    def set_coordinates(self, coords):
+        """
+        Sets the field coordinates and calculates the centroid latitude and longitude.
+        """
+        self.field_coords = coords
+        field_polygon = Polygon(self.field_coords)
+        self.latitude = field_polygon.centroid.y
+        self.longitude = field_polygon.centroid.x
+
+    def fetch_weather_data(self, start_date, end_date):
+        """
+        Fetches weather data, automatically switching between Historical and Ensemble Forecast APIs.
+        """
+        # Ensure dates are date objects
+        if isinstance(start_date, pd.Timestamp):
+            start_date = start_date.date()
+        if isinstance(end_date, pd.Timestamp):
+            end_date = end_date.date()
+
+        today = datetime.date.today()
+        
+        dfs = []
+
+        # 1. Historical Data (if start_date < today)
+        if start_date < today:
+            hist_end = min(end_date, today - datetime.timedelta(days=1))
+            if start_date <= hist_end:
+                print(f"Fetching historical data from {start_date} to {hist_end}...")
+                try:
+                    hist_df = self._fetch_historical_api(start_date, hist_end)
+                    dfs.append(hist_df)
+                except Exception as e:
+                    print(f"Error fetching historical data: {e}")
+
+        # 2. Forecast Data (if end_date >= today)
+        if end_date >= today:
+            print(f"Fetching forecast data from {today} to {end_date}...")
+            try:
+                # Calculate needed forecast days
+                days_needed = (end_date - today).days + 1
+                # API supports up to 35 days for ensemble
+                forecast_days = min(max(days_needed, 1), 35) 
+                
+                forecast_df = self._fetch_ensemble_forecast_api(forecast_days)
+                
+                # Filter for requested range
+                forecast_df = forecast_df[
+                    (forecast_df["ds"].dt.date >= today) & 
+                    (forecast_df["ds"].dt.date <= end_date)
+                ]
+                dfs.append(forecast_df)
+            except Exception as e:
+                print(f"Error fetching forecast data: {e}")
+
+        if not dfs:
+            print("Warning: No weather data fetched.")
+            return pd.DataFrame(columns=["ds", "temp", "humidity", "rainfall"])
+
+        final_df = pd.concat(dfs, ignore_index=True)
+        final_df = final_df.sort_values("ds").reset_index(drop=True)
+        
+        # Aggregate to daily if not already (The helpers return daily)
+        # But we need to ensure unique dates in case of overlap
+        final_df = final_df.drop_duplicates(subset=["ds"], keep="last")
+        
+        # Filter to ensure exact range (handling timezone spillover)
+        final_df = final_df[
+            (final_df["ds"].dt.date >= start_date) & 
+            (final_df["ds"].dt.date <= end_date)
+        ]
+        
+        return final_df.reset_index(drop=True)
+
+    def _fetch_historical_api(self, start_date, end_date):
+        url = "https://archive-api.open-meteo.com/v1/archive"
+        params = {
+            "latitude": self.latitude,
+            "longitude": self.longitude,
+            "start_date": start_date.strftime("%Y-%m-%d"),
+            "end_date": end_date.strftime("%Y-%m-%d"),
+            "daily": ["temperature_2m_mean", "rain_sum"], # We need these for consistency check, but we use hourly aggregated
+            "hourly": ["temperature_2m", "relative_humidity_2m", "rain"],
+            "timezone": "auto",
+        }
+        
+        responses = self.openmeteo.weather_api(url, params=params)
+        response = responses[0]
+        
+        hourly = response.Hourly()
+        hourly_data = {
+            "date": pd.date_range(
+                start=pd.to_datetime(hourly.Time(), unit="s", utc=True),
+                end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True),
+                freq=pd.Timedelta(seconds=hourly.Interval()),
+                inclusive="left",
+            ),
+            "temperature_2m": hourly.Variables(0).ValuesAsNumpy(),
+            "relative_humidity_2m": hourly.Variables(1).ValuesAsNumpy(),
+            "rain": hourly.Variables(2).ValuesAsNumpy(),
+        }
+        
+        hourly_df = pd.DataFrame(data=hourly_data)
+        
+        # Daily aggregation
+        hourly_df["ds"] = hourly_df["date"].dt.floor("D").dt.tz_convert(None)
+        daily_weather = (
+            hourly_df.groupby("ds")
+            .agg(
+                temp=("temperature_2m", "mean"),
+                humidity=("relative_humidity_2m", "mean"),
+                rainfall=("rain", "sum"),
+            )
+            .reset_index()
+        )
+        return daily_weather
+
+    def _fetch_ensemble_forecast_api(self, forecast_days):
+        url = "https://ensemble-api.open-meteo.com/v1/ensemble"
+        params = {
+            "latitude": self.latitude,
+            "longitude": self.longitude,
+            "hourly": ["temperature_2m", "relative_humidity_2m", "rain"],
+            "models": ["ecmwf_ifs025", "gfs025", "icon_global", "icon_seamless", "gem_global", "bom_access_global_ensemble"],
+            "timezone": "auto",
+            "forecast_days": forecast_days,
+        }
+        responses = self.openmeteo.weather_api(url, params=params)
+        
+        # We will aggregate all models and members into a single mean
+        all_hourly_dfs = []
+        
+        for response in responses:
+            hourly = response.Hourly()
+            
+            # Helper to extract all members for a variable
+            def get_members(variable_type):
+                # variable_type is an enum from Variable class? 
+                # The snippet uses filter on hourly.Variables()
+                # We need to map the snippet logic here.
+                vars_list = [hourly.Variables(i) for i in range(hourly.VariablesLength())]
+                return [v for v in vars_list if v.Variable() == variable_type]
+
+            # We need Variable enum. 
+            # Note: The snippet imports Variable. 
+            # We need to check if Variable.temperature is correct mapping for "temperature_2m"
+            # In snippet: Variable.temperature and Altitude() == 2
+            
+            temp_vars = [v for v in [hourly.Variables(i) for i in range(hourly.VariablesLength())] 
+                         if v.Variable() == Variable.temperature and v.Altitude() == 2]
+            
+            rh_vars = [v for v in [hourly.Variables(i) for i in range(hourly.VariablesLength())] 
+                       if v.Variable() == Variable.relative_humidity and v.Altitude() == 2]
+            
+            rain_vars = [v for v in [hourly.Variables(i) for i in range(hourly.VariablesLength())] 
+                         if v.Variable() == Variable.rain]
+
+            # Create a DF for this model
+            dates = pd.date_range(
+                start=pd.to_datetime(hourly.Time(), unit="s", utc=True),
+                end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=True),
+                freq=pd.Timedelta(seconds=hourly.Interval()),
+                inclusive="left",
+            )
+            
+            model_df = pd.DataFrame({"date": dates})
+            
+            # Average members for this model
+            if temp_vars:
+                temps = np.stack([v.ValuesAsNumpy() for v in temp_vars])
+                model_df["temp"] = np.mean(temps, axis=0)
+            else:
+                model_df["temp"] = np.nan
+                
+            if rh_vars:
+                rhs = np.stack([v.ValuesAsNumpy() for v in rh_vars])
+                model_df["humidity"] = np.mean(rhs, axis=0)
+            else:
+                model_df["humidity"] = np.nan
+                
+            if rain_vars:
+                rains = np.stack([v.ValuesAsNumpy() for v in rain_vars])
+                model_df["rainfall"] = np.mean(rains, axis=0)
+            else:
+                model_df["rainfall"] = 0
+                
+            all_hourly_dfs.append(model_df)
+            
+        # Concatenate all models
+        full_hourly = pd.concat(all_hourly_dfs, ignore_index=True)
+        
+        # Group by date and take mean across all models
+        full_hourly = full_hourly.groupby("date").mean().reset_index()
+        
+        # Daily aggregation
+        full_hourly["ds"] = full_hourly["date"].dt.floor("D").dt.tz_convert(None)
+        daily_weather = (
+            full_hourly.groupby("ds")
+            .agg(
+                temp=("temp", "mean"),
+                humidity=("humidity", "mean"),
+                rainfall=("rainfall", "sum"),
+            )
+            .reset_index()
+        )
+        
+        return daily_weather
+
+    def preprocess_data(self, df):
+        """
+        Preprocesses the input dataframe:
+        1. Fetches historical weather
+        2. Creates lag/diff features
+        3. Scales data
+        """
+        df = df.copy()
+        df["ds"] = pd.to_datetime(df["ds"])
+        df = df.sort_values("ds").reset_index(drop=True)
+        df["unique_id"] = "VV" 
+
+        # 1. Fetch Historical Weather
+        start_date = df["ds"].min().date()
+        end_date = df["ds"].max().date()
+        print(f"Fetching historical weather from {start_date} to {end_date}...")
+        weather_df = self.fetch_weather_data(start_date, end_date)
+        
+        df = df.merge(weather_df, on="ds", how="left")
+        df[self.futr_exog_list] = df[self.futr_exog_list].ffill().bfill()
+
+        # 2. Feature Engineering (Lags/Diffs)
+        df["lag1"] = df["y"].shift(1)
+        df["diff"] = df["y"].diff()
+        
+        df = df.dropna().reset_index(drop=True)
+
+        # 3. Scaling
+        df["y"] = self.y_scaler.fit_transform(df[["y"]])
+        
+        all_exog = self.hist_exog_list + self.futr_exog_list
+        df[all_exog] = self.exog_scaler.fit_transform(df[all_exog])
+
+        return df
+
+    def tune_and_predict(self, csv_path, field_coords, target_col="y", output_file="auto_tuned_predictions.csv", num_samples=10):
+        """
+        Runs AutoNHITS tuning and predicts the next 20 days.
+        """
+        print(f"Setting coordinates to: {field_coords}")
+        self.set_coordinates(field_coords)
+
+        # 1. Load Data
+        print("Loading data...")
+        df = pd.read_csv(csv_path)
+        
+        # Rename target column to 'y' if it exists
+        if target_col in df.columns:
+            df = df.rename(columns={target_col: "y"})
+            
+        if "ds" not in df.columns or "y" not in df.columns:
+            raise ValueError(f"CSV must contain 'ds' and '{target_col}' (mapped to 'y') columns.")
+        
+        # 2. Preprocess
+        print("Preprocessing data...")
+        train_df = self.preprocess_data(df)
+
+        # 3. Prepare Future Dataframe
+        last_date = train_df["ds"].max()
+        # Ensure we predict for at least 30 days as per user request "next 1 month"
+        # Data frequency is 5 Days, so 30 days / 5 = 6 periods
+        prediction_days = 6 
+        # Start 5 days after the last training date
+        future_dates = pd.date_range(start=last_date + pd.Timedelta(days=5), periods=prediction_days, freq="5D")
+        future_df = pd.DataFrame({"ds": future_dates, "unique_id": "VV"})
+
+        # 4. Fetch Future Weather
+        print("Fetching future weather forecast...")
+        # The new fetch_weather_data handles the logic automatically
+        weather_future = self.fetch_weather_data(future_dates[0].date(), future_dates[-1].date())
+
+        future_df = future_df.merge(weather_future, on="ds", how="left")
+        future_df[self.futr_exog_list] = future_df[self.futr_exog_list].fillna(0)
+
+        # 5. Auto Model Definition
+        import ray.tune as tune
+        print(f"Initializing AutoNHITS model (tuning with {num_samples} samples)...")
+        
+        # Define a custom search space (same as auto_tuning_testing.py)
+        config = {
+            "input_size": tune.choice([60, 90, 120]),              # Lookback window
+            "learning_rate": tune.loguniform(1e-4, 1e-2),          # Learning rate
+            "n_blocks": tune.choice([[1, 1, 1], [3, 3, 3]]),       # Depth
+            "mlp_units": tune.choice([                             # Width
+                [[64, 64], [64, 64], [64, 64]],
+                [[512, 512], [512, 512], [512, 512]]
+            ]),
+            "n_pool_kernel_size": tune.choice([                    # Pooling
+                [2, 2, 1], 
+                [4, 4, 2],
+                [8, 4, 1]
+            ]),
+            "n_freq_downsample": tune.choice([                     # Downsampling
+                [2, 1, 1],
+                [4, 2, 1],
+                [8, 4, 1]
+            ])
+        }
+
+        # AutoNHITS configuration
+        auto_nhits = AutoNHITS(
+            h=prediction_days, # Horizon (6 steps = 30 days)
+            loss=MAE(),
+            config=config, 
+            search_alg=None, # Use default search algorithm (HyperOpt)
+            num_samples=num_samples, # Number of trials
+            cpus=1,
+            gpus=0, # Set to 1 if GPU available
+            verbose=True,
+            alias="AutoNHITS"
+        )
+
+        nf = NeuralForecast(models=[auto_nhits], freq="5D")
+
+        # 6. Train (Tune) and Predict
+        print("Tuning and Training model...")
+        nf.fit(df=train_df)
+        
+        # Get best config
+        # The model inside nf.models[0] is the trained AutoNHITS
+        # It should have 'results' or 'best_config' attribute after fitting?
+        # Actually, AutoNHITS replaces itself with the best model found or wraps it.
+        
+        print("Predicting...")
+        # Prepare future exogenous features
+        # Note: We need to scale them!
+        X_futr = future_df[self.futr_exog_list].values
+        
+        # Get indices of futr_exog in the scaler
+        # self.exog_scaler was fitted on [hist_exog + futr_exog]
+        # futr_exog are the last columns, starting after hist_exog
+        start_idx = len(self.hist_exog_list)
+        futr_indices = [start_idx + i for i in range(len(self.futr_exog_list))]
+        
+        means = self.exog_scaler.mean_[futr_indices]
+        scales = self.exog_scaler.scale_[futr_indices]
+        
+        X_futr_scaled = (X_futr - means) / scales
+        future_df[self.futr_exog_list] = X_futr_scaled
+
+        preds_df = nf.predict(futr_df=future_df)
+        
+        # 7. Inverse Scale Predictions
+        y_pred_scaled = preds_df["AutoNHITS"].values.reshape(-1, 1)
+        y_pred = self.y_scaler.inverse_transform(y_pred_scaled).flatten()
+        
+        result = pd.DataFrame({
+            "ds": preds_df["ds"].values,
+            "predicted_y": y_pred
+        })
+        
+        # 8. Plot results
+        result.to_csv(output_file, index=False)
+        print(f"\nPredictions saved to {output_file}")
+        
+        return result
+        
+        try:
+            from plot_predictions import plot_predictions
+            print("\nGenerating plot...")
+            plot_predictions(csv_path, output_file)
+        except Exception as e:
+            print(f"Warning: Could not generate plot: {e}")
+            
+        return result
+
+if __name__ == "__main__":
+    # Define the input file path here
+    target_file = "vh_data_structured.csv"
+    
+    print(f"Using input file: {target_file}")
+    
+    if os.path.exists(target_file):
+        predictor = AutoTimeSeriesPredictor()
+        
+        # Example coordinates
+        coords = [
+            (77.2090, 28.6139),
+            (77.2100, 28.6139),
+            (77.2100, 28.6149),
+            (77.2090, 28.6149),
+            (77.2090, 28.6139),
+        ]
+        
+        try:
+            predictions = predictor.tune_and_predict(target_file, field_coords=coords, num_samples=30) # 30 samples for better results
+            print("\nPredictions for the next 20 days:")
+            print(predictions)
+        except Exception as e:
+            print(f"An error occurred: {e}")
+    else:
+        print(f"File not found: {target_file}")

prediction_pipeline.py

@@ -0,0 +1,145 @@
+"""
+Prediction Pipeline
+Orchestrates satellite data fetching and time series prediction.
+"""
+
+import os
+import pandas as pd
+from satellite_pipeline import SatelliteFetcher
+from auto_tuning_predictor import AutoTimeSeriesPredictor
+
+class PredictionOrchestrator:
+    """
+    Orchestrates the entire pipeline:
+    1. Fetch Satellite Data (SAR & Sentinel-2)
+    2. Generate Predictions for ALL bands using AutoNHITS
+    """
+    
+    def __init__(self, polygon_coords):
+        """
+        Initialize the orchestrator.
+        
+        Args:
+            polygon_coords (list): List of (lon, lat) tuples defining the polygon.
+        """
+        self.polygon_coords = polygon_coords
+        self.sar_csv = 'sar_data.csv'
+        self.sentinel2_csv = 'sentinel2_data.csv'
+        
+        self.sar_pred_csv = 'sar_predictions.csv'
+        self.sentinel2_pred_csv = 'sentinel2_predictions.csv'
+
+    def run(self):
+        """Run the full pipeline."""
+        print("="*60)
+        print("STARTING PREDICTION PIPELINE")
+        print("="*60)
+        
+        # 1. Fetch Satellite Data
+        print("\n[1/2] Fetching Satellite Data...")
+        fetcher = SatelliteFetcher(self.polygon_coords)
+        fetcher.run_all()
+        # print("Skipping fetch for verification (using existing CSVs)...")
+        
+        # 2. Generate Predictions
+        print("\n[2/2] Generating Predictions...")
+        predictor = AutoTimeSeriesPredictor()
+        
+        # --- SAR Prediction ---
+        print("\n" + "-"*30)
+        print("Predicting SAR Data (All Bands)")
+        print("-"*30)
+        self._predict_all_bands(
+            predictor, 
+            input_csv=self.sar_csv, 
+            output_csv=self.sar_pred_csv
+        )
+
+        # --- Sentinel-2 Prediction ---
+        print("\n" + "-"*30)
+        print("Predicting Sentinel-2 Data (All Bands)")
+        print("-"*30)
+        self._predict_all_bands(
+            predictor, 
+            input_csv=self.sentinel2_csv, 
+            output_csv=self.sentinel2_pred_csv
+        )
+            
+        print("\n" + "="*60)
+        print("PIPELINE COMPLETE")
+        print(f"SAR Predictions: {self.sar_pred_csv}")
+        print(f"Sentinel-2 Predictions: {self.sentinel2_pred_csv}")
+        print("="*60)
+
+    def _predict_all_bands(self, predictor, input_csv, output_csv):
+        """
+        Helper to predict all numerical columns in a CSV.
+        """
+        if not os.path.exists(input_csv):
+            print(f"❌ Input file not found: {input_csv}")
+            return
+
+        df = pd.read_csv(input_csv)
+        if 'ds' not in df.columns:
+            print(f"❌ 'ds' column missing in {input_csv}")
+            return
+            
+        # Identify target columns (all except 'ds')
+        target_cols = [c for c in df.columns if c != 'ds']
+        print(f"Found {len(target_cols)} targets: {target_cols}")
+        
+        all_preds = []
+        
+        for col in target_cols:
+            print(f"\n>> Predicting {col}...")
+            try:
+                # We use a temporary output file for individual band predictions
+                temp_out = f"temp_pred_{col}.csv"
+                
+                pred_df = predictor.tune_and_predict(
+                    csv_path=input_csv,
+                    field_coords=self.polygon_coords,
+                    target_col=col,
+                    output_file=temp_out,
+                    num_samples=5 
+                )
+                
+                # Rename predicted column to the band name
+                pred_df = pred_df.rename(columns={'predicted_y': col})
+                
+                if not all_preds:
+                    all_preds.append(pred_df)
+                else:
+                    # Merge on 'ds'
+                    all_preds.append(pred_df[['ds', col]])
+                
+                # Clean up temp file
+                if os.path.exists(temp_out):
+                    os.remove(temp_out)
+                    
+            except Exception as e:
+                print(f"❌ Failed to predict {col}: {e}")
+        
+        # Merge all predictions
+        if all_preds:
+            final_df = all_preds[0]
+            for i in range(1, len(all_preds)):
+                final_df = final_df.merge(all_preds[i], on='ds', how='outer')
+            
+            final_df.to_csv(output_csv, index=False)
+            print(f"✓ Saved combined predictions to {output_csv}")
+        else:
+            print("⚠ No predictions generated.")
+
+if __name__ == "__main__":
+    # Example Usage
+    POLYGON = [
+        (75.829, 30.229),
+        (75.831, 30.229),
+        (75.831, 30.231),
+        (75.829, 30.231),
+        (75.829, 30.229)
+    ]
+    
+    pipeline = PredictionOrchestrator(POLYGON)
+    pipeline.run()

requirements.txt

@@ -0,0 +1,17 @@
+fastapi==0.104.1
+uvicorn[standard]==0.24.0
+pydantic==2.5.2
+numpy>=1.24.0
+pandas>=2.0.0
+shapely>=2.0.0
+python-dotenv>=1.0.0
+sentinelhub>=3.9.0
+scikit-learn>=1.3.0
+matplotlib>=3.7.0
+neuralforecast>=1.6.0
+openmeteo-requests>=1.2.0
+requests-cache>=1.1.0
+retry-requests>=2.0.0
+openmeteo-sdk>=1.7.0
+ray[tune]>=2.7.0
+torch>=2.0.0

satellite_pipeline.py

@@ -0,0 +1,311 @@
+"""
+Satellite Data Pipeline
+Fetches both Sentinel-1 (SAR) and Sentinel-2 (Optical) data for a given polygon.
+Outputs two separate CSV files: `sar_data.csv` and `sentinel2_data.csv`.
+"""
+
+import os
+import datetime
+import numpy as np
+import pandas as pd
+from shapely.geometry import Polygon
+from dotenv import load_dotenv
+
+from sentinelhub import (
+    SHConfig, 
+    SentinelHubRequest,
+    SentinelHubCatalog,
+    DataCollection, 
+    MimeType, 
+    BBox, 
+    CRS,
+    bbox_to_dimensions,
+    Geometry
+)
+
+# Load environment variables
+load_dotenv()
+
+class SatelliteFetcher:
+    """
+    A class to fetch and process satellite data (SAR and Optical) for a specific area of interest.
+    """
+    
+    def __init__(self, polygon_coords):
+        """
+        Initialize the fetcher.
+        
+        Args:
+            polygon_coords (list): List of (lon, lat) tuples defining the polygon.
+        """
+        self.polygon_coords = polygon_coords
+        self.start_date = '2020-01-01'
+        self.end_date = datetime.date.today().strftime('%Y-%m-%d')
+        
+        # Configuration
+        self.resolution = 10
+        self.max_cloud_cover = 20.0
+        
+        # Setup Sentinel Hub
+        self.config = self._setup_config()
+        self.geometry, self.bbox, self.size = self._setup_geometry()
+        
+    def _setup_config(self):
+        """Configure Sentinel Hub credentials."""
+        config = SHConfig()
+        config.sh_client_id = os.environ.get('SH_CLIENT_ID')
+        config.sh_client_secret = os.environ.get('SH_CLIENT_SECRET')
+        
+        if not config.sh_client_id:
+            config.sh_client_id = "sh-4c23abb9-6263-4a2c-bdba-a4ff6b84bdfb"
+        if not config.sh_client_secret:
+            config.sh_client_secret = "iZ4gexrfiWQpopHAGYlBeEMj9J8DAZtD"
+
+        config.sh_base_url = 'https://sh.dataspace.copernicus.eu'
+        config.sh_token_url = 'https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token'
+        config.save("cdse")
+        
+        return config
+
+    def _setup_geometry(self):
+        """Setup geometry, bbox, and size from polygon coordinates."""
+        shapely_poly = Polygon(self.polygon_coords)
+        geometry = Geometry(shapely_poly, crs=CRS.WGS84)
+        bbox = geometry.bbox
+        size = bbox_to_dimensions(bbox, resolution=self.resolution)
+        
+        print(f"AOI Setup:")
+        print(f"  Polygon: {self.polygon_coords}")
+        print(f"  BBox: {bbox}")
+        print(f"  Size: {size}")
+        print(f"  Time Range: {self.start_date} to {self.end_date}")
+        
+        return geometry, bbox, size
+
+    def fetch_sar_data(self, output_csv='sar_data.csv'):
+        """
+        Fetch Sentinel-1 SAR data (VV, VH) and save to CSV.
+        """
+        print("\n" + "="*40)
+        print("FETCHING SENTINEL-1 SAR DATA")
+        print("="*40)
+        
+        S1 = DataCollection.define(
+            name="SENTINEL1_IW_CDSE",
+            api_id="sentinel-1-grd",
+            service_url="https://sh.dataspace.copernicus.eu"
+        )
+        
+        evalscript = """
+        //VERSION=3
+        function setup() {
+          return {
+            input: ["VV", "VH", "dataMask"],
+            output: [
+              { id: "VV", bands: 1, sampleType: "FLOAT32" },
+              { id: "VH", bands: 1, sampleType: "FLOAT32" }
+            ]
+          };
+        }
+        
+        function evaluatePixel(sample) {
+          let vv_db = (sample.VV > 0) ? 10 * Math.log(sample.VV) / Math.LN10 : -9999;
+          let vh_db = (sample.VH > 0) ? 10 * Math.log(sample.VH) / Math.LN10 : -9999;
+          return { VV: [vv_db], VH: [vh_db] };
+        }
+        """
+        
+        print("🔍 Searching catalog...")
+        catalog = SentinelHubCatalog(config=self.config)
+        results = catalog.search(
+            collection=S1,
+            geometry=self.geometry,
+            time=(self.start_date, self.end_date),
+            filter="sar:instrument_mode = 'IW'"
+        )
+        
+        scenes = list(results)
+        dates = sorted(list(set([scene['properties']['datetime'].split('T')[0] for scene in scenes])))
+        print(f"✓ Found {len(dates)} unique dates.")
+        
+        all_records = []
+        
+        print("📥 Downloading and Processing...")
+        for date_str in dates:
+            dt = datetime.datetime.strptime(date_str, "%Y-%m-%d")
+            next_day = (dt + datetime.timedelta(days=1)).strftime("%Y-%m-%d")
+            
+            request = SentinelHubRequest(
+                evalscript=evalscript,
+                input_data=[
+                    SentinelHubRequest.input_data(
+                        data_collection=S1,
+                        time_interval=(date_str, next_day),
+                        mosaicking_order="leastRecent",
+                        other_args={"processing": {"backCoeff": "GAMMA0_TERRAIN", "orthorectify": True}}
+                    )
+                ],
+                responses=[
+                    SentinelHubRequest.output_response('VV', MimeType.TIFF),
+                    SentinelHubRequest.output_response('VH', MimeType.TIFF)
+                ],
+                geometry=self.geometry,
+                bbox=self.bbox,
+                size=self.size,
+                config=self.config
+            )
+
+            try:
+                data = request.get_data()
+                if data and len(data) > 0:
+                    data_dict = data[0]
+                    if 'VV.tif' in data_dict and 'VH.tif' in data_dict:
+                        vv_arr = data_dict['VV.tif']
+                        vh_arr = data_dict['VH.tif']
+                        
+                        valid_mask = (vv_arr > -9999) & (vh_arr > -9999)
+                        
+                        if np.any(valid_mask):
+                            vv_mean = np.mean(vv_arr[valid_mask])
+                            vh_mean = np.mean(vh_arr[valid_mask])
+                            
+                            all_records.append({
+                                'ds': date_str,
+                                'VV_mean_dB': round(vv_mean, 4),
+                                'VH_mean_dB': round(vh_mean, 4)
+                            })
+                            print(f"  ✓ {date_str}: VV={vv_mean:.2f}, VH={vh_mean:.2f}")
+                        else:
+                            print(f"  ⚠ {date_str}: No valid pixels")
+            except Exception as e:
+                print(f"  ❌ Error {date_str}: {e}")
+                
+        df = pd.DataFrame(all_records)
+        if not df.empty:
+            df = df.sort_values('ds').reset_index(drop=True)
+            df.to_csv(output_csv, index=False)
+            print(f"\n💾 Saved SAR data to: {output_csv}")
+        else:
+            print("\n⚠ No SAR data fetched.")
+
+    def fetch_sentinel2_data(self, output_csv='sentinel2_data.csv'):
+        """
+        Fetch Sentinel-2 L2A data (Bands + SCL) and save to CSV.
+        """
+        print("\n" + "="*40)
+        print("FETCHING SENTINEL-2 OPTICAL DATA")
+        print("="*40)
+        
+        S2 = DataCollection.define(
+            name="SENTINEL2_L2A_CDSE",
+            api_id="sentinel-2-l2a",
+            service_url="https://sh.dataspace.copernicus.eu",
+            collection_type="Sentinel-2",
+            is_timeless=False
+        )
+        
+        evalscript = """
+        //VERSION=3
+        function setup() {
+            return {
+                input: [{
+                    bands: ["B01", "B02", "B03", "B04", "B05", "B06", "B07", "B08", "B8A", "B09", "B11", "B12", "SCL", "dataMask"],
+                    units: "DN"
+                }],
+                output: {
+                    bands: 13,
+                    sampleType: "FLOAT32"
+                }
+            };
+        }
+    
+        function evaluatePixel(sample) {
+            if (sample.dataMask == 0) {
+                return [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
+            }
+            return [
+                sample.B01 / 10000, sample.B02 / 10000, sample.B03 / 10000, sample.B04 / 10000,
+                sample.B05 / 10000, sample.B06 / 10000, sample.B07 / 10000, sample.B08 / 10000,
+                sample.B8A / 10000, sample.B09 / 10000, sample.B11 / 10000, sample.B12 / 10000,
+                sample.SCL
+            ];
+        }
+        """
+        
+        print("🔍 Searching catalog...")
+        catalog = SentinelHubCatalog(config=self.config)
+        search_iterator = catalog.search(
+            S2,
+            geometry=self.geometry,
+            time=(self.start_date, self.end_date),
+            filter=f'eo:cloud_cover < {self.max_cloud_cover}'
+        )
+        
+        dates = sorted(list(set([item['properties']['datetime'] for item in search_iterator])))
+        print(f"✓ Found {len(dates)} available scenes.")
+        
+        results = []
+        band_names = ["B01", "B02", "B03", "B04", "B05", "B06", "B07", "B08", "B8A", "B09", "B11", "B12"]
+        
+        print("📥 Downloading and Processing...")
+        for i, date_str in enumerate(dates):
+            request = SentinelHubRequest(
+                evalscript=evalscript,
+                input_data=[SentinelHubRequest.input_data(
+                    data_collection=S2,
+                    time_interval=(date_str, date_str)
+                )],
+                responses=[SentinelHubRequest.output_response('default', MimeType.TIFF)],
+                geometry=self.geometry,
+                bbox=self.bbox,
+                size=self.size,
+                config=self.config
+            )
+            
+            try:
+                data = request.get_data()[0]
+                scl = data[:, :, 12]
+                valid_mask = (scl != 0)
+                
+                if np.any(valid_mask):
+                    means = {}
+                    for b_idx, b_name in enumerate(band_names):
+                        means[b_name] = np.mean(data[:, :, b_idx][valid_mask])
+                    
+                    means['ds'] = date_str.split('T')[0]
+                    results.append(means)
+                    print(f"  ✓ {date_str[:10]}")
+                else:
+                    print(f"  ⚠ {date_str[:10]} (No valid pixels)")
+            except Exception as e:
+                print(f"  ❌ Error {date_str[:10]}: {e}")
+
+        df = pd.DataFrame(results)
+        if not df.empty:
+            cols = ['ds'] + [c for c in df.columns if c != 'ds']
+            df = df[cols]
+            df.to_csv(output_csv, index=False)
+            print(f"\n💾 Saved Sentinel-2 data to: {output_csv}")
+        else:
+            print("\n⚠ No Sentinel-2 data fetched.")
+
+    def run_all(self):
+        """Run both pipelines."""
+        self.fetch_sar_data()
+        self.fetch_sentinel2_data()
+
+
+if __name__ == "__main__":
+    # Example Usage
+    # PAU Experimental Farm, Punjab, India
+    POLYGON = [
+        (75.829, 30.229),
+        (75.831, 30.229),
+        (75.831, 30.231),
+        (75.829, 30.231),
+        (75.829, 30.229)
+    ]
+    
+    # Initialize and Run
+    fetcher = SatelliteFetcher(POLYGON)
+    fetcher.run_all()