feat: integrate machine learning forecasting pipeline and unify settings & dashboard under Sikomo theme

Dockerfile

@@ -23,7 +23,12 @@ WORKDIR /app
 
 # Install minimal OS build dependencies
 RUN apt-get update && apt-get install -y \
+    libgomp1 \
     gcc \
+    g++ \
+    make \
+    python3-dev \
+    curl \
     && rm -rf /var/lib/apt/lists/*
 
 # Install python dependencies as user
@@ -34,8 +39,8 @@ RUN pip install --no-cache-dir --upgrade pip \
 # Copy backend python packages
 COPY --chown=user backend/ ./
 
-# Establish web serving static destination directory
-RUN mkdir -p /app/static && chown -R user:user /app/static
+# Establish web serving static destination directory and model directory
+RUN mkdir -p /app/static /app/models && chown -R user:user /app/static /app/models
 COPY --chown=user --from=builder /app/frontend/out /app/static
 
 # Enforce secure runner execution ownership context

backend/app/config.py

@@ -1,33 +1,79 @@
+# config.py — Centralized Configuration using pydantic-settings
 from pydantic_settings import BaseSettings
 from functools import lru_cache
+from typing import Optional
+import os
 
 class Settings(BaseSettings):
-    # Database
-    MYSQL_HOST: str
-    MYSQL_PORT: int = 3306
-    MYSQL_DATABASE: str
-    MYSQL_USER: str
-    MYSQL_PASSWORD: str
-    
-    # API
+    # Database (support both MYSQL_ prefix and DB_ prefix)
+    MYSQL_HOST: Optional[str] = None
+    MYSQL_PORT: Optional[int] = None
+    MYSQL_DATABASE: Optional[str] = None
+    MYSQL_USER: Optional[str] = None
+    MYSQL_PASSWORD: Optional[str] = None
+
+    DB_HOST: Optional[str] = None
+    DB_PORT: Optional[int] = None
+    DB_NAME: Optional[str] = None
+    DB_USER: Optional[str] = None
+    DB_PASSWORD: Optional[str] = None
+
+    # API Auth & Configuration
     API_PORT: int = 8000
     API_HOST: str = "0.0.0.0"
-    
-    # Scheduler
+    API_SECRET_KEY: str = "your-random-secret"
+    DASHBOARD_PASSWORD: str = "Bandulan112"
+
+    # Default Schedules
     DEFAULT_SCHEDULE: str = "0 8 * * *"
-    DEFAULT_SCHEDULES: str = "0 8 * * *"  # comma-separated if multiple
-    
+    DEFAULT_SCHEDULES: str = "0 8 * * *"
+
     # CORS
     FRONTEND_URL: str = "http://localhost:3000"
-    
+
     @property
     def DATABASE_URL(self) -> str:
-        return f"mysql+pymysql://{self.MYSQL_USER}:{self.MYSQL_PASSWORD}@{self.MYSQL_HOST}:{self.MYSQL_PORT}/{self.MYSQL_DATABASE}"
-    
+        # Fallback mappings
+        host = self.MYSQL_HOST or self.DB_HOST or os.getenv("MYSQL_HOST") or os.getenv("DB_HOST") or "localhost"
+        port = self.MYSQL_PORT or self.DB_PORT or os.getenv("MYSQL_PORT") or os.getenv("DB_PORT") or 3306
+        db = self.MYSQL_DATABASE or self.DB_NAME or os.getenv("MYSQL_DATABASE") or os.getenv("DB_NAME") or "sikomo_db"
+        user = self.MYSQL_USER or self.DB_USER or os.getenv("MYSQL_USER") or os.getenv("DB_USER") or "root"
+        password = self.MYSQL_PASSWORD or self.DB_PASSWORD or os.getenv("MYSQL_PASSWORD") or os.getenv("DB_PASSWORD") or ""
+        return f"mysql+pymysql://{user}:{password}@{host}:{port}/{db}"
+
     class Config:
         env_file = ".env"
         extra = "ignore"
 
 @lru_cache()
-def get_settings():
-    return Settings()
+def get_settings() -> Settings:
+    return Settings()
+
+class RuntimeConfig:
+    """Runtime ML Configuration editable via dashboard without server restart."""
+    def __init__(self):
+        self.optuna_trials: int = int(os.getenv("OPTUNA_TRIALS", "30"))
+        self.model_dir: str = os.getenv("MODEL_DIR", "models")
+        self.forecast_days: int = int(os.getenv("FORECAST_DAYS", "7"))
+        self.default_schedules: str = os.getenv("DEFAULT_FORECAST_SCHEDULES", "0 1 * * *")
+
+    def to_dict(self) -> dict:
+        return {
+            "optuna_trials": self.optuna_trials,
+            "model_dir": self.model_dir,
+            "forecast_days": self.forecast_days,
+            "default_schedules": self.default_schedules
+        }
+
+    def update(self, key: str, value: str):
+        allowed = {
+            "optuna_trials": lambda v: setattr(self, "optuna_trials", int(v)),
+            "model_dir": lambda v: setattr(self, "model_dir", v),
+            "forecast_days": lambda v: setattr(self, "forecast_days", int(v)),
+            "default_schedules": lambda v: setattr(self, "default_schedules", v),
+        }
+        if key not in allowed:
+            raise ValueError(f"Key '{key}' tidak diizinkan untuk diubah.")
+        allowed[key](value)
+
+runtime_config = RuntimeConfig()

backend/app/database.py

@@ -1,8 +1,12 @@
-from sqlalchemy import create_engine
+# database.py — MySQL Connection & CRUD Operations for Scraper and Forecast
+import pandas as pd
+import sqlalchemy
+from sqlalchemy import create_engine, text
 from app.config import get_settings
 
 settings = get_settings()
 
+# Global Singleton DB Engine
 engine = create_engine(
     settings.DATABASE_URL,
     pool_pre_ping=True,
@@ -10,4 +14,294 @@ engine = create_engine(
 )
 
 def get_db_engine():
-    return engine
+    return engine
+
+def get_engine():
+    """Compatibility alias for forecast operations."""
+    return engine
+
+# ── READ OPERATIONS ───────────────────────────────────────────────────────────
+
+def get_all_komoditas():
+    """Ambil semua komoditas aktif."""
+    return pd.read_sql(
+        "SELECT id, nama, slug, unit, volatile, volatilitas_skor FROM komoditas WHERE is_active=1",
+        engine
+    )
+
+def get_harga_harian(komoditas_id: int, days: int = 730):
+    """Ambil data harga harian untuk satu komoditas (default 2 tahun terakhir)."""
+    query = f"""
+        SELECT hh.tanggal AS date, p.nama AS pasar_nama, hh.harga
+        FROM harga_harian hh
+        JOIN pasar p ON p.id = hh.pasar_id
+        WHERE hh.komoditas_id = {komoditas_id}
+          AND hh.tanggal >= DATE_SUB(CURDATE(), INTERVAL {days} DAY)
+        ORDER BY hh.tanggal ASC
+    """
+    df_raw = pd.read_sql(query, engine)
+    if df_raw.empty:
+        return pd.DataFrame()
+    # Pivot → wide format
+    df = df_raw.pivot_table(
+        index='date', columns='pasar_nama', values='harga'
+    ).reset_index()
+    df.columns.name = None
+    df['date'] = pd.to_datetime(df['date'])
+    df = df.sort_values('date').reset_index(drop=True)
+    return df
+
+def get_pasar_list(komoditas_id: int):
+    """Ambil daftar pasar yang punya data untuk komoditas ini."""
+    return pd.read_sql(f"""
+        SELECT DISTINCT p.id, p.nama
+        FROM harga_harian hh
+        JOIN pasar p ON p.id = hh.pasar_id
+        WHERE hh.komoditas_id = {komoditas_id}
+    """, engine)
+
+# ── WRITE OPERATIONS ──────────────────────────────────────────────────────────
+
+def save_model_ml(data: dict):
+    """Upsert ke tabel model_ml."""
+    with engine.begin() as conn:
+        conn.execute(text("""
+            INSERT INTO model_ml (
+                komoditas_id, nama_model, versi, deskripsi, file_path,
+                mape, rmse, mae, r2_score, confidence_level,
+                stabilitas, status_validasi, catatan_validasi,
+                tanggal_training, tanggal_evaluasi, is_active,
+                created_at, updated_at
+            ) VALUES (
+                :komoditas_id, :nama_model, :versi, :deskripsi, :file_path,
+                :mape, :rmse, :mae, :r2_score, :confidence_level,
+                :stabilitas, :status_validasi, :catatan_validasi,
+                :tanggal_training, :tanggal_evaluasi, 1, NOW(), NOW()
+            )
+            ON DUPLICATE KEY UPDATE
+                nama_model       = VALUES(nama_model),
+                versi            = VALUES(versi),
+                deskripsi        = VALUES(deskripsi),
+                file_path        = VALUES(file_path),
+                mape             = VALUES(mape),
+                rmse             = VALUES(rmse),
+                mae              = VALUES(mae),
+                r2_score         = VALUES(r2_score),
+                confidence_level = VALUES(confidence_level),
+                stabilitas       = VALUES(stabilitas),
+                status_validasi  = VALUES(status_validasi),
+                catatan_validasi = VALUES(catatan_validasi),
+                tanggal_evaluasi = VALUES(tanggal_evaluasi),
+                updated_at       = NOW()
+        """), data)
+
+    with engine.connect() as conn:
+        return conn.execute(
+            text("SELECT id FROM model_ml WHERE komoditas_id=:kid ORDER BY updated_at DESC LIMIT 1"),
+            {'kid': data['komoditas_id']}
+        ).scalar()
+
+def save_hasil_prediksi(predictions: list, komoditas_id: int, pasar_id, model_id: int, meta: dict):
+    """Hapus prediksi lama & insert 7 baris baru."""
+    with engine.begin() as conn:
+        conn.execute(text("""
+            DELETE FROM hasil_prediksi
+            WHERE komoditas_id = :kid AND pasar_id = :pid
+              AND tanggal_target >= CURDATE()
+        """), {'kid': komoditas_id, 'pid': pasar_id})
+
+        for p in predictions:
+            conn.execute(text("""
+                INSERT INTO hasil_prediksi (
+                    komoditas_id, pasar_id, model_id,
+                    tanggal_prediksi, tanggal_target,
+                    harga_prediksi, confidence_level,
+                    model_name, mape, rmse, created_at
+                ) VALUES (
+                    :kid, :pid, :model_id,
+                    CURDATE(), :tanggal_target,
+                    :harga_prediksi, :confidence_level,
+                    :model_name, :mape, :rmse, NOW()
+                )
+            """), {
+                'kid'             : komoditas_id,
+                'pid'             : pasar_id,
+                'model_id'        : model_id,
+                'tanggal_target'  : p['tanggal'],
+                'harga_prediksi'  : p['harga_prediksi'],
+                'confidence_level': meta['confidence_level'],
+                'model_name'      : meta['nama_model'],
+                'mape'            : meta['mape'],
+                'rmse'            : meta['rmse'],
+            })
+
+def save_ringkasan_prediksi(data: dict, komoditas_id: int, model_id: int):
+    """Upsert ringkasan prediksi."""
+    with engine.begin() as conn:
+        conn.execute(text("""
+            INSERT INTO ringkasan_prediksi (
+                komoditas_id, model_id,
+                harga_min, harga_max, tren, confidence_level,
+                status_analisis, deskripsi_status,
+                tanggal_mulai, tanggal_akhir, created_at
+            ) VALUES (
+                :komoditas_id, :model_id,
+                :harga_min, :harga_max, :tren, :confidence_level,
+                :status_analisis, :deskripsi_status,
+                :tanggal_mulai, :tanggal_akhir, NOW()
+            )
+            ON DUPLICATE KEY UPDATE
+                harga_min        = VALUES(harga_min),
+                harga_max        = VALUES(harga_max),
+                tren             = VALUES(tren),
+                confidence_level = VALUES(confidence_level),
+                status_analisis  = VALUES(status_analisis),
+                deskripsi_status = VALUES(deskripsi_status),
+                tanggal_mulai    = VALUES(tanggal_mulai),
+                tanggal_akhir    = VALUES(tanggal_akhir)
+        """), {**data, 'komoditas_id': komoditas_id, 'model_id': model_id})
+
+def save_insight_prediksi(insights: list, komoditas_id: int, model_id: int):
+    """Hapus insight lama & insert baru."""
+    with engine.begin() as conn:
+        conn.execute(text(
+            "DELETE FROM insight_prediksi WHERE komoditas_id = :kid"
+        ), {'kid': komoditas_id})
+        for ins in insights:
+            conn.execute(text("""
+                INSERT INTO insight_prediksi (
+                    komoditas_id, model_id, konten, tipe, ikon,
+                    urutan, is_active, created_at, updated_at
+                ) VALUES (
+                    :kid, :model_id, :konten, :tipe, :ikon,
+                    :urutan, 1, NOW(), NOW()
+                )
+            """), {
+                'kid'     : komoditas_id,
+                'model_id': model_id,
+                'konten'  : ins['konten'],
+                'tipe'    : ins['tipe'],
+                'ikon'    : ins['ikon'],
+                'urutan'  : ins['urutan'],
+            })
+
+def update_komoditas_volatilitas(komoditas_id: int, volatile: int, skor: float):
+    with engine.begin() as conn:
+        conn.execute(text("""
+            UPDATE komoditas SET volatile=:v, volatilitas_skor=:s, updated_at=NOW()
+            WHERE id=:kid
+        """), {'v': volatile, 's': skor, 'kid': komoditas_id})
+
+def save_all_model_results(all_results: dict, best_meta: dict, komoditas_id: int, pasar_id: int):
+    """Save ALL model results to model_ml (not just best). Best gets is_active=1."""
+    best_name = best_meta.get('nama_model', '')
+    with engine.begin() as conn:
+        # Deactivate old models for this komoditas
+        conn.execute(text(
+            "UPDATE model_ml SET is_active=0 WHERE komoditas_id=:kid"
+        ), {'kid': komoditas_id})
+        # Insert each model result
+        for model_name, metrics in all_results.items():
+            is_best = 1 if model_name == best_name or f"{model_name}_Tuned" == best_name else 0
+            
+            # Hitung stabilitas sederhana untuk kolom ENUM agar tidak error
+            mape_val = metrics.get('mape', 0)
+            stabilitas_label = (
+                'optimal' if mape_val < 2 else 
+                'baik' if mape_val < 5 else 
+                'cukup' if mape_val < 10 else 
+                'perlu_retrain'
+            )
+
+            conn.execute(text("""
+                INSERT INTO model_ml (
+                    komoditas_id, nama_model, versi, deskripsi, file_path,
+                    mape, rmse, mae, r2_score, confidence_level,
+                    stabilitas, status_validasi, catatan_validasi,
+                    tanggal_training, tanggal_evaluasi, is_active,
+                    created_at, updated_at
+                ) VALUES (
+                    :komoditas_id, :nama_model, '1.0', :deskripsi, '',
+                    :mape, :rmse, :mae, :r2_score, 0,
+                    :stabilitas, 'terverifikasi', '',
+                    CURDATE(), CURDATE(), :is_active, NOW(), NOW()
+                )
+            """), {
+                'komoditas_id': komoditas_id,
+                'nama_model': model_name,
+                'deskripsi': f"Model {model_name} untuk komoditas ID {komoditas_id}",
+                'mape': round(mape_val, 4),
+                'rmse': round(metrics.get('rmse', 0), 4),
+                'mae': round(metrics.get('mae', 0), 4),
+                'r2_score': round(metrics.get('r2', 0), 4),
+                'stabilitas': stabilitas_label,
+                'is_active': is_best,
+            })
+
+# ── READ OPERATIONS FOR FRONTEND API ──────────────────────────────────────────
+
+def get_prediksi_data(komoditas_id: int):
+    """Ambil semua data yang dibutuhkan frontend untuk satu komoditas."""
+    komoditas = pd.read_sql(f"""
+        SELECT k.*, kk.nama as kategori_nama
+        FROM komoditas k
+        JOIN kategori_komoditas kk ON kk.id = k.kategori_id
+        WHERE k.id = {komoditas_id}
+    """, engine).to_dict('records')
+    komoditas = komoditas[0] if komoditas else None
+
+    harga_terkini = pd.read_sql(f"""
+        SELECT hh.harga, hh.tanggal, p.nama as pasar_nama, p.id as pasar_id
+        FROM harga_harian hh
+        JOIN pasar p ON p.id = hh.pasar_id
+        WHERE hh.komoditas_id = {komoditas_id}
+        ORDER BY hh.tanggal DESC LIMIT 1
+    """, engine).to_dict('records')
+    harga_terkini = harga_terkini[0] if harga_terkini else None
+
+    # Best (active) model
+    model = pd.read_sql(f"""
+        SELECT * FROM model_ml WHERE komoditas_id={komoditas_id} AND is_active=1
+        ORDER BY updated_at DESC LIMIT 1
+    """, engine).to_dict('records')
+    model = model[0] if model else None
+
+    # ALL models for comparison (latest run)
+    all_models = pd.read_sql(f"""
+        SELECT nama_model, mape, rmse, mae, r2_score, is_active,
+               tanggal_training, created_at
+        FROM model_ml WHERE komoditas_id={komoditas_id}
+          AND DATE(created_at) = (
+            SELECT DATE(MAX(created_at)) FROM model_ml WHERE komoditas_id={komoditas_id}
+          )
+        ORDER BY rmse ASC
+    """, engine).to_dict('records')
+
+    ringkasan = pd.read_sql(f"""
+        SELECT * FROM ringkasan_prediksi WHERE komoditas_id={komoditas_id}
+        ORDER BY created_at DESC LIMIT 1
+    """, engine).to_dict('records')
+    ringkasan = ringkasan[0] if ringkasan else None
+
+    hasil = pd.read_sql(f"""
+        SELECT tanggal_target, harga_prediksi, confidence_level
+        FROM hasil_prediksi
+        WHERE komoditas_id={komoditas_id} AND tanggal_target >= CURDATE()
+        ORDER BY tanggal_target ASC
+    """, engine).to_dict('records')
+
+    insights = pd.read_sql(f"""
+        SELECT * FROM insight_prediksi
+        WHERE komoditas_id={komoditas_id} AND is_active=1
+        ORDER BY urutan ASC
+    """, engine).to_dict('records')
+
+    return {
+        'komoditas'     : komoditas,
+        'harga_terkini' : harga_terkini,
+        'model'         : model,
+        'all_models'    : all_models,
+        'ringkasan'     : ringkasan,
+        'prediksi_7hari': hasil,
+        'insights'      : insights,
+    }

backend/app/logger.py

@@ -1,8 +1,10 @@
+# logger.py — In-memory log handler for scraper and forecast log separation
 import logging
 from datetime import datetime
 from typing import List, Dict, Optional
 
 class MemoryLogHandler(logging.Handler):
+    """Stores log entries in memory for API log streaming."""
     def __init__(self, capacity: int = 500):
         super().__init__()
         self.capacity = capacity
@@ -10,7 +12,6 @@ class MemoryLogHandler(logging.Handler):
 
     def emit(self, record):
         try:
-            # We want just the message or formatted message
             msg = record.getMessage()
             log_entry = {
                 "timestamp": datetime.fromtimestamp(record.created).isoformat(),
@@ -33,12 +34,42 @@ class MemoryLogHandler(logging.Handler):
     def clear_logs(self):
         self.logs.clear()
 
-memory_handler = MemoryLogHandler()
+# Separated handlers
+scraper_memory_handler = MemoryLogHandler(capacity=500)
+forecast_memory_handler = MemoryLogHandler(capacity=500)
 
-def get_app_logger(name: str):
-    logger = logging.getLogger(f"app.{name}")
+def get_app_logger(name: str) -> logging.Logger:
+    """Default app logger for scraping operations."""
+    logger = logging.getLogger(f"app.scraper.{name}")
     logger.setLevel(logging.INFO)
-    # Ensure memory handler is added
-    if not any(isinstance(h, MemoryLogHandler) for h in logger.handlers):
-        logger.addHandler(memory_handler)
+    
+    # Add memory handler
+    if not any(h is scraper_memory_handler for h in logger.handlers):
+        logger.addHandler(scraper_memory_handler)
+        
+    # Add console stream handler
+    if not any(isinstance(h, logging.StreamHandler) and not isinstance(h, MemoryLogHandler) for h in logger.handlers):
+        console = logging.StreamHandler()
+        console.setLevel(logging.INFO)
+        console.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] scraper.%(name)s: %(message)s"))
+        logger.addHandler(console)
+        
+    return logger
+
+def get_forecast_logger(name: str) -> logging.Logger:
+    """Dedicated app logger for forecasting operations."""
+    logger = logging.getLogger(f"app.forecast.{name}")
+    logger.setLevel(logging.INFO)
+    
+    # Add memory handler
+    if not any(h is forecast_memory_handler for h in logger.handlers):
+        logger.addHandler(forecast_memory_handler)
+        
+    # Add console stream handler
+    if not any(isinstance(h, logging.StreamHandler) and not isinstance(h, MemoryLogHandler) for h in logger.handlers):
+        console = logging.StreamHandler()
+        console.setLevel(logging.INFO)
+        console.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] forecast.%(name)s: %(message)s"))
+        logger.addHandler(console)
+        
     return logger

backend/app/main.py

@@ -1,20 +1,41 @@
-from fastapi import FastAPI, HTTPException, Query
-from fastapi.middleware.cors import CORSMiddleware
+# main.py — FastAPI: Refactored with modular config, logger, scheduler
+import os
+import threading
+from datetime import datetime
+from typing import Optional, List
+from fastapi import FastAPI, HTTPException, Depends, BackgroundTasks, Query
 from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from pydantic import BaseModel
-from typing import Optional
-from app.scheduler import scraper_scheduler
-from app.scraper import scraping_hari_ini, upload_to_mysql
-from app.database import get_db_engine
-from app.config import get_settings
-from app.logger import get_app_logger, memory_handler
-import os
 
-logger = get_app_logger("main")
+# Modular App Imports
+from app.config import get_settings, runtime_config
+from app.logger import get_app_logger, get_forecast_logger, scraper_memory_handler, forecast_memory_handler
+from app.scheduler import scraper_scheduler, forecast_scheduler
+from app.scraper import scraping_hari_ini, upload_to_mysql, stop_event as scraper_stop_event
+from app.database import (
+    get_db_engine,
+    get_all_komoditas,
+    get_harga_harian,
+    get_pasar_list,
+    get_prediksi_data,
+    save_model_ml,
+    save_hasil_prediksi,
+    save_ringkasan_prediksi,
+    save_insight_prediksi,
+    update_komoditas_volatilitas,
+    save_all_model_results,
+)
+from app.ml_pipeline import run_pipeline
+from sqlalchemy import text
+
 settings = get_settings()
+logger = get_app_logger("main")
+forecast_logger = get_forecast_logger("main")
 
-app = FastAPI(title="Sikomo Scraper API", version="1.3.0")
+app = FastAPI(title="Sikomo Scraper & ML Portal API", version="2.0.0")
 
 app.add_middleware(
     CORSMiddleware,
@@ -24,85 +45,200 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-class ScheduleAdd(BaseModel):
-    cron_expression: str
-    label: Optional[str] = ""
+# ── Authentication Dependency ──────────────────────────────────────────────────
+security = HTTPBearer()
 
-class ScheduleUpdate(BaseModel):
-    cron_expression: str
+def verify_token(creds: HTTPAuthorizationCredentials = Depends(security)):
+    if creds.credentials != settings.API_SECRET_KEY:
+        raise HTTPException(status_code=401, detail="Unauthorized")
+    return creds.credentials
 
-class CustomScrapeRequest(BaseModel):
-    date: str  # Format YYYY-MM-DD
+# ── ML Forecasting State ──────────────────────────────────────────────────────
+running_forecast_tasks: dict = {}  # komoditas_id -> True/False
+forecast_stop_event = threading.Event()  # For aborting forecast
+
+def run_forecast_for_komoditas(komoditas_id: int, komoditas_nama: str):
+    """Run forecast for a single komoditas. Tracks running state."""
+    if running_forecast_tasks.get(komoditas_id):
+        forecast_logger.warning(f"⚠️ {komoditas_nama} sudah sedang berjalan, skip.")
+        return
+
+    running_forecast_tasks[komoditas_id] = True
+    try:
+        if forecast_stop_event.is_set():
+            forecast_logger.warning(f"⛔ Forecast dihentikan sebelum {komoditas_nama}.")
+            return
+
+        df = get_harga_harian(komoditas_id)
+        if df.empty or len(df) < 60:
+            forecast_logger.warning(f"⚠️ {komoditas_nama}: data kurang ({len(df)} baris)")
+            return
+
+        price_cols = [c for c in df.columns if c != 'date']
+        target_col = price_cols[0]
+
+        pasar_df = get_pasar_list(komoditas_id)
+        pasar_id = int(pasar_df.iloc[0]['id']) if not pasar_df.empty else 1
+
+        def log_cb(msg):
+            forecast_logger.info(f"[{komoditas_nama}] {msg}")
+
+        result = run_pipeline(
+            komoditas_id=komoditas_id,
+            komoditas_nama=komoditas_nama,
+            df=df,
+            target_market=target_col,
+            pasar_id=pasar_id,
+            n_trials=runtime_config.optuna_trials,
+            forecast_days=runtime_config.forecast_days,
+            log_cb=log_cb,
+        )
+
+        # Save results to DB
+        meta = result['metadata']
+        labels = result['labels']
+        preds = result['predictions']
+        all_model_results = result.get('model_comparison', {})
+
+        # Save all model results
+        save_all_model_results(all_model_results, meta, komoditas_id, pasar_id)
+
+        # Save best model specifically
+        model_id = save_model_ml(meta)
+        save_hasil_prediksi(preds, komoditas_id, pasar_id, model_id, meta)
+        save_ringkasan_prediksi({
+            'harga_min': labels['harga_min'],
+            'harga_max': labels['harga_max'],
+            'tren': labels['tren'],
+            'confidence_level': labels['confidence_level'],
+            'status_analisis': labels['status_analisis']['judul'],
+            'deskripsi_status': labels['status_analisis']['deskripsi'],
+            'tanggal_mulai': preds[0]['tanggal'],
+            'tanggal_akhir': preds[-1]['tanggal'],
+        }, komoditas_id, model_id)
+        save_insight_prediksi(labels['insights'], komoditas_id, model_id)
+        cv = meta.get('data_quality', {}).get('cv', 0)
+        update_komoditas_volatilitas(komoditas_id, 1 if cv >= 5 else 0, cv)
 
+        forecast_logger.info(f"✅ {komoditas_nama} selesai! Best: {meta['nama_model']} MAPE={meta['mape']:.2f}%")
+
+    except Exception as e:
+        forecast_logger.error(f"❌ {komoditas_nama} gagal: {e}")
+    finally:
+        running_forecast_tasks[komoditas_id] = False
+
+def auto_forecast_all():
+    """Run forecast for ALL komoditas (scheduled or manual run-all)."""
+    forecast_logger.info("🚀 Auto forecast semua komoditas dimulai...")
+    forecast_stop_event.clear()
+    try:
+        komoditas_list = get_all_komoditas()
+        total = len(komoditas_list)
+        for idx, (_, row) in enumerate(komoditas_list.iterrows()):
+            if forecast_stop_event.is_set():
+                forecast_logger.warning("⛔ Forecast dihentikan oleh user.")
+                break
+            forecast_logger.info(f"📦 [{idx+1}/{total}] Memproses {row['nama']}...")
+            run_forecast_for_komoditas(int(row['id']), row['nama'])
+        forecast_logger.info("🏁 Auto forecast selesai.")
+    except Exception as e:
+        forecast_logger.error(f"❌ Auto forecast error: {e}")
+
+# Register forecast job with scheduler
+forecast_scheduler.set_forecast_job(auto_forecast_all)
+
+# ── API Startup & Shutdown ─────────────────────────────────────────────────────
 @app.on_event("startup")
 async def startup_event():
-    logger.info("🎉 Memulai Sikomo Scraper API Server...")
+    logger.info("🎉 Memulai Sikomo Scraper & Forecast API Server...")
+    os.makedirs(runtime_config.model_dir, exist_ok=True)
     scraper_scheduler.start_all()
+    forecast_scheduler.start_all(default_schedules=runtime_config.default_schedules)
     logger.info("🚀 API Server siap menerima permintaan.")
 
 @app.on_event("shutdown")
 async def shutdown_event():
     scraper_scheduler.stop_all()
+    forecast_scheduler.stop_all()
     logger.info("🛑 API Server dimatikan.")
 
-# --- API Endpoints ---
+# ── Request Validation Models ──────────────────────────────────────────────────
+class ScheduleAdd(BaseModel):
+    cron_expression: str
+    label: Optional[str] = ""
 
-@app.get("/status")
-async def get_status():
-    return scraper_scheduler.get_status()
+class CustomScrapeRequest(BaseModel):
+    date: str  # Format YYYY-MM-DD
 
-@app.get("/debug/static")
-def debug_static():
-    import os
-    files = []
-    for root, dirs, filenames in os.walk("/app/static"):
-        for f in filenames:
-            files.append(os.path.relpath(os.path.join(root, f), "/app/static"))
-    return {"files": sorted(files)}
+class ForecastRequest(BaseModel):
+    komoditas_id: int
+
+class InsightUpdate(BaseModel):
+    konten: str
+    tipe: str
+    ikon: str
+    urutan: int
+
+class RingkasanUpdate(BaseModel):
+    status_analisis: Optional[str] = None
+    deskripsi_status: Optional[str] = None
+    tren: Optional[str] = None
+
+class ModelUpdate(BaseModel):
+    deskripsi: Optional[str] = None
+    catatan_validasi: Optional[str] = None
+
+class RuntimeConfigUpdate(BaseModel):
+    key: str
+    value: str
+
+# ══════════════════════════════════════════════════════════════════════════════
+# ENDPOINTS
+# ══════════════════════════════════════════════════════════════════════════════
 
-@app.get("/logs")
-async def get_logs(limit: int = Query(100, le=500), level: Optional[str] = None):
-    return memory_handler.get_logs(limit=limit, level=level)
+# ── Auth ──────────────────────────────────────────────────────────────────────
+@app.post("/auth/login")
+def login(body: dict):
+    if body.get('password') != settings.DASHBOARD_PASSWORD:
+        raise HTTPException(status_code=401, detail="Password salah")
+    return {"token": settings.API_SECRET_KEY}
 
-@app.get("/schedules")
-async def list_schedules():
+# ── Portal Scraper API ────────────────────────────────────────────────────────
+@app.get("/scraper/status", dependencies=[Depends(verify_token)])
+async def get_scraper_status():
+    return scraper_scheduler.get_status()
+
+@app.get("/scraper/logs", dependencies=[Depends(verify_token)])
+async def get_scraper_logs(limit: int = Query(100, le=500), level: Optional[str] = None):
+    return scraper_memory_handler.get_logs(limit=limit, level=level)
+
+@app.get("/scraper/schedules", dependencies=[Depends(verify_token)])
+async def get_scraper_schedules():
     return scraper_scheduler.list_schedules()
 
-@app.post("/schedules/add")
-async def add_schedule(schedule: ScheduleAdd):
+@app.post("/scraper/schedules/add", dependencies=[Depends(verify_token)])
+async def add_scraper_schedule(schedule: ScheduleAdd):
     try:
         job_id = scraper_scheduler.add_schedule(schedule.cron_expression, label=schedule.label)
         return {
             "success": True, 
             "job_id": job_id,
-            "message": f"Jadwal berhasil ditambahkan: {schedule.cron_expression}"
+            "message": f"Jadwal scraper berhasil ditambahkan: {schedule.cron_expression}"
         }
     except Exception as e:
         raise HTTPException(status_code=400, detail=str(e))
 
-@app.delete("/schedules/{job_id}/remove")
-async def remove_schedule(job_id: str):
+@app.delete("/scraper/schedules/{job_id}/remove", dependencies=[Depends(verify_token)])
+async def remove_scraper_schedule(job_id: str):
     try:
         scraper_scheduler.remove_schedule(job_id)
-        return {"success": True, "message": "Jadwal berhasil dihapus"}
+        return {"success": True, "message": "Jadwal scraper berhasil dihapus"}
     except KeyError:
         raise HTTPException(status_code=404, detail="Jadwal tidak ditemukan")
     except Exception as e:
         raise HTTPException(status_code=400, detail=str(e))
 
-# Backward compatibility for single schedule update if UI still uses it
-@app.post("/schedule/update")
-async def update_schedule(schedule: ScheduleUpdate):
-    try:
-        # Clear existing and add new
-        scraper_scheduler.stop_all()
-        scraper_scheduler.start_all()
-        job_id = scraper_scheduler.add_schedule(schedule.cron_expression, label="Jadwal Utama")
-        return {"success": True, "message": f"Jadwal diupdate ke: {schedule.cron_expression}"}
-    except Exception as e:
-        raise HTTPException(status_code=400, detail=str(e))
-
-@app.post("/scrape/manual")
+@app.post("/scraper/run/manual", dependencies=[Depends(verify_token)])
 def manual_scrape():
     logger.info("⚡ Permintaan manual scraping dipicu via API...")
     try:
@@ -111,12 +247,12 @@ def manual_scrape():
         stats = upload_to_mysql(df, engine)
         return {"success": True, "stats": stats}
     except Exception as e:
-        logger.error(f"❌ Manual scrape gagal: {e}")
+        logger.error(f"❌ Manual scraping gagal: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
-@app.post("/scrape/custom")
+@app.post("/scraper/run/custom", dependencies=[Depends(verify_token)])
 def custom_scrape(req: CustomScrapeRequest):
-    logger.info(f"⚡ Permintaan manual scraping custom untuk tanggal {req.date} dipicu via API...")
+    logger.info(f"⚡ Permintaan scraping custom untuk tanggal {req.date} dipicu via API...")
     try:
         from datetime import datetime
         target_date = datetime.strptime(req.date, "%Y-%m-%d")
@@ -127,28 +263,212 @@ def custom_scrape(req: CustomScrapeRequest):
     except ValueError:
         raise HTTPException(status_code=400, detail="Format tanggal salah. Gunakan YYYY-MM-DD.")
     except Exception as e:
-        logger.error(f"❌ Custom scrape gagal: {e}")
+        logger.error(f"❌ Scraping custom gagal: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
-@app.post("/scrape/stop")
+@app.post("/scraper/run/stop", dependencies=[Depends(verify_token)])
 def stop_manual_scrape():
-    from app.scraper import stop_event
     logger.warning("🛑 Menerima permintaan penghentian scraping manual...")
-    stop_event.set()
-    return {"success": True, "message": "Permintaan penghentian dikirim"}
+    scraper_stop_event.set()
+    return {"success": True, "message": "Permintaan penghentian scraper dikirim"}
 
-@app.post("/schedule/start")
-async def start_scheduler():
+@app.post("/scraper/schedule/start", dependencies=[Depends(verify_token)])
+async def start_scraper_scheduler():
     scraper_scheduler.start_all()
-    return {"success": True, "message": "Scheduler otomatis diaktifkan"}
+    return {"success": True, "message": "Scheduler scraper diaktifkan"}
 
-@app.post("/schedule/stop")
-async def stop_scheduler():
+@app.post("/scraper/schedule/stop", dependencies=[Depends(verify_token)])
+async def stop_scraper_scheduler():
     scraper_scheduler.stop_all()
-    return {"success": True, "message": "Scheduler otomatis dimatikan"}
+    return {"success": True, "message": "Scheduler scraper dimatikan"}
+
+
+# ── Portal Forecast API ───────────────────────────────────────────────────────
+@app.get("/forecast/status", dependencies=[Depends(verify_token)])
+def get_forecast_status():
+    sched = forecast_scheduler.get_status()
+    return {
+        **sched,
+        "running_tasks": {k: v for k, v in running_forecast_tasks.items() if v},
+        "any_forecast_running": any(running_forecast_tasks.values()),
+    }
+
+@app.get("/forecast/logs", dependencies=[Depends(verify_token)])
+def get_forecast_logs(limit: int = Query(200, le=500), level: Optional[str] = None):
+    return forecast_memory_handler.get_logs(limit=limit, level=level)
+
+@app.post("/forecast/clear-logs", dependencies=[Depends(verify_token)])
+def clear_forecast_logs():
+    forecast_memory_handler.clear_logs()
+    return {"success": True, "message": "Log forecast berhasil dihapus."}
+
+@app.get("/forecast/schedules", dependencies=[Depends(verify_token)])
+def list_forecast_schedules():
+    return forecast_scheduler.list_schedules()
+
+@app.post("/forecast/schedules/add", dependencies=[Depends(verify_token)])
+def add_forecast_schedule(schedule: ScheduleAdd):
+    try:
+        job_id = forecast_scheduler.add_schedule(
+            schedule.cron_expression, label=schedule.label
+        )
+        return {
+            "success": True,
+            "job_id": job_id,
+            "message": f"Jadwal forecast berhasil ditambahkan: {schedule.cron_expression}",
+        }
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@app.delete("/forecast/schedules/{job_id}/remove", dependencies=[Depends(verify_token)])
+def remove_forecast_schedule(job_id: str):
+    try:
+        forecast_scheduler.remove_schedule(job_id)
+        return {"success": True, "message": "Jadwal forecast berhasil dihapus."}
+    except KeyError:
+        raise HTTPException(status_code=404, detail="Jadwal tidak ditemukan")
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@app.post("/forecast/schedule/start", dependencies=[Depends(verify_token)])
+def start_forecast_scheduler():
+    forecast_scheduler.start_all(default_schedules=runtime_config.default_schedules)
+    return {"success": True, "message": "Scheduler forecast diaktifkan."}
+
+@app.post("/forecast/schedule/stop", dependencies=[Depends(verify_token)])
+def stop_forecast_scheduler():
+    forecast_scheduler.stop_all()
+    return {"success": True, "message": "Scheduler forecast dimatikan."}
+
+@app.get("/forecast/komoditas", dependencies=[Depends(verify_token)])
+def list_forecast_komoditas():
+    df = get_all_komoditas()
+    return df.to_dict('records')
+
+@app.get("/forecast/prediksi/{komoditas_id}", dependencies=[Depends(verify_token)])
+def get_forecast_prediksi(komoditas_id: int):
+    return get_prediksi_data(komoditas_id)
+
+@app.post("/forecast/run", dependencies=[Depends(verify_token)])
+def run_forecast_manual(req: ForecastRequest, bg: BackgroundTasks):
+    """Run forecast for a SINGLE komoditas."""
+    if running_forecast_tasks.get(req.komoditas_id):
+        raise HTTPException(409, f"Komoditas ID {req.komoditas_id} sedang diproses.")
+
+    komoditas_list = get_all_komoditas()
+    row = komoditas_list[komoditas_list['id'] == req.komoditas_id]
+    if row.empty:
+        raise HTTPException(404, "Komoditas tidak ditemukan")
+    nama = row.iloc[0]['nama']
+    forecast_stop_event.clear()
+    bg.add_task(run_forecast_for_komoditas, req.komoditas_id, nama)
+    return {"message": f"Forecast untuk {nama} dimulai di background."}
+
+@app.post("/forecast/run-all", dependencies=[Depends(verify_token)])
+def run_forecast_all(bg: BackgroundTasks):
+    """Run forecast for ALL komoditas."""
+    if any(running_forecast_tasks.values()):
+        raise HTTPException(409, "Ada forecast yang sedang berjalan.")
+    forecast_stop_event.clear()
+    bg.add_task(auto_forecast_all)
+    return {"message": "Forecast semua komoditas dimulai."}
+
+@app.post("/forecast/stop", dependencies=[Depends(verify_token)])
+def stop_forecast():
+    """Abort running forecast."""
+    forecast_logger.warning("🛑 Menerima permintaan penghentian forecast...")
+    forecast_stop_event.set()
+    return {"success": True, "message": "Permintaan penghentian dikirim."}
+
+@app.get("/forecast/settings", dependencies=[Depends(verify_token)])
+def get_runtime_settings():
+    return runtime_config.to_dict()
+
+@app.post("/forecast/settings", dependencies=[Depends(verify_token)])
+def update_runtime_settings(body: RuntimeConfigUpdate):
+    try:
+        runtime_config.update(body.key, body.value)
+        return {"message": f"{body.key} berhasil diperbarui ke '{body.value}'."}
+    except ValueError as e:
+        raise HTTPException(400, str(e))
+
+# ── CRUD Forecast Insights, Ringkasan, Model ──────────────────────────────────
+@app.get("/forecast/insight/{komoditas_id}", dependencies=[Depends(verify_token)])
+def get_insights(komoditas_id: int):
+    engine = get_db_engine()
+    with engine.connect() as conn:
+        rows = conn.execute(
+            text("SELECT * FROM insight_prediksi WHERE komoditas_id=:kid ORDER BY urutan"),
+            {'kid': komoditas_id}
+        ).fetchall()
+    return [dict(r._mapping) for r in rows]
+
+@app.put("/forecast/insight/{insight_id}", dependencies=[Depends(verify_token)])
+def update_insight(insight_id: int, body: InsightUpdate):
+    engine = get_db_engine()
+    with engine.begin() as conn:
+        conn.execute(text("""
+            UPDATE insight_prediksi
+            SET konten=:konten, tipe=:tipe, ikon=:ikon, urutan=:urutan, updated_at=NOW()
+            WHERE id=:id
+        """), {**body.dict(), 'id': insight_id})
+    return {"message": "Insight diperbarui."}
+
+@app.post("/forecast/insight/{komoditas_id}", dependencies=[Depends(verify_token)])
+def add_insight(komoditas_id: int, body: InsightUpdate):
+    engine = get_db_engine()
+    with engine.begin() as conn:
+        conn.execute(text("""
+            INSERT INTO insight_prediksi (komoditas_id, konten, tipe, ikon, urutan, is_active, created_at, updated_at)
+            VALUES (:kid, :konten, :tipe, :ikon, :urutan, 1, NOW(), NOW())
+        """), {**body.dict(), 'kid': komoditas_id})
+    return {"message": "Insight ditambahkan."}
+
+@app.delete("/forecast/insight/{insight_id}", dependencies=[Depends(verify_token)])
+def delete_insight(insight_id: int):
+    engine = get_db_engine()
+    with engine.begin() as conn:
+        conn.execute(text("DELETE FROM insight_prediksi WHERE id=:id"), {'id': insight_id})
+    return {"message": "Insight dihapus."}
+
+@app.put("/forecast/ringkasan/{komoditas_id}", dependencies=[Depends(verify_token)])
+def update_ringkasan(komoditas_id: int, body: RingkasanUpdate):
+    engine = get_db_engine()
+    updates = {k: v for k, v in body.dict().items() if v is not None}
+    if not updates:
+        raise HTTPException(400, "Tidak ada field yang diupdate.")
+    set_clause = ', '.join([f"{k}=:{k}" for k in updates])
+    with engine.begin() as conn:
+        conn.execute(
+            text(f"UPDATE ringkasan_prediksi SET {set_clause} WHERE komoditas_id=:kid ORDER BY created_at DESC LIMIT 1"),
+            {**updates, 'kid': komoditas_id}
+        )
+    return {"message": "Ringkasan diperbarui."}
+
+@app.put("/forecast/model-ml/{komoditas_id}", dependencies=[Depends(verify_token)])
+def update_model_ml(komoditas_id: int, body: ModelUpdate):
+    engine = get_db_engine()
+    updates = {k: v for k, v in body.dict().items() if v is not None}
+    if not updates:
+        raise HTTPException(400, "Tidak ada field yang diupdate.")
+    set_clause = ', '.join([f"{k}=:{k}" for k in updates])
+    with engine.begin() as conn:
+        conn.execute(
+            text(f"UPDATE model_ml SET {set_clause}, updated_at=NOW() WHERE komoditas_id=:kid AND is_active=1"),
+            {**updates, 'kid': komoditas_id}
+        )
+    return {"message": "Model ML diperbarui."}
+
+# ── Debug Endpoints ───────────────────────────────────────────────────────────
+@app.get("/debug/static", dependencies=[Depends(verify_token)])
+def debug_static():
+    files = []
+    for root, dirs, filenames in os.walk("/app/static"):
+        for f in filenames:
+            files.append(os.path.relpath(os.path.join(root, f), "/app/static"))
+    return {"files": sorted(files)}
 
-# --- Static Files / SPA Serving ---
-# Mount static folder if exists (in Docker production or local test)
+# ── SPA Serving fallback ──────────────────────────────────────────────────────
 static_dirs = [
     "/app/static",
     os.path.join(os.path.dirname(os.path.dirname(__file__)), "static"),
@@ -166,6 +486,6 @@ if not mounted:
     @app.get("/")
     async def root_fallback():
         return {
-            "message": "Sikomo Scraper API Server beroperasi normal.",
+            "message": "Sikomo Scraper & Forecast API Server beroperasi normal.",
             "note": "Frontend static files belum di-build/mount ke direktori static."
         }

backend/app/ml_pipeline.py

@@ -0,0 +1,557 @@
+# ml_pipeline.py — Full ML pipeline: cleaning → training → forecasting → labeling
+import os, json, pickle, warnings, logging
+import pandas as pd
+import numpy as np
+from datetime import date
+from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
+from sklearn.model_selection import TimeSeriesSplit
+import lightgbm as lgb
+import xgboost as xgb
+from statsmodels.tsa.arima.model import ARIMA
+from statsmodels.tsa.statespace.sarimax import SARIMAX
+import optuna
+optuna.logging.set_verbosity(optuna.logging.WARNING)
+warnings.filterwarnings('ignore')
+logging.getLogger('cmdstanpy').setLevel(logging.WARNING)
+logging.getLogger('prophet').setLevel(logging.WARNING)
+
+MODEL_DIR = os.getenv('MODEL_DIR', 'models')
+os.makedirs(MODEL_DIR, exist_ok=True)
+
+# ══════════════════════════════════════════════════════════════════════════════
+# 1. DATA QUALITY & CLEANING
+# ══════════════════════════════════════════════════════════════════════════════
+
+def analyze_data_quality(df, col):
+    total, missing = len(df), df[col].isna().sum()
+    gap_lengths, cur = [], 0
+    for v in df[col].isna():
+        if v: cur += 1
+        else:
+            if cur > 0: gap_lengths.append(cur)
+            cur = 0
+    max_gap  = max(gap_lengths) if gap_lengths else 0
+    outliers = (df[col].pct_change().abs() > 0.5).sum()
+    mean, std = df[col].mean(), df[col].std()
+    cv = (std / mean * 100) if mean > 0 else 0
+    return {
+        'total_records': total,
+        'missing'      : int(missing),
+        'missing_pct'  : round(missing / total * 100, 2),
+        'max_gap_days' : max_gap,
+        'outliers'     : int(outliers),
+        'cv'           : round(cv, 2),
+        'volatility'   : 'rendah' if cv < 2 else ('sedang' if cv < 5 else 'tinggi'),
+        'mean_price'   : round(mean, 2),
+        'std_price'    : round(std, 2),
+    }
+
+def clean_price_data(df, col, quality_info):
+    df   = df.copy()
+    df[f'{col}_was_missing'] = df[col].isna().astype(int)
+    temp = df[col].copy()
+    mask = temp.isna()
+    gap_id    = (mask != mask.shift()).cumsum()
+    gap_sizes = mask.groupby(gap_id).transform('sum')
+
+    temp = temp.fillna(method='ffill', limit=3)
+    med_mask = mask & (gap_sizes > 3) & (gap_sizes <= 7)
+    ti = temp.interpolate(method='linear', limit=7)
+    temp[med_mask] = ti[med_mask]
+
+    long_mask = mask & (gap_sizes > 7)
+    if long_mask.any():
+        rolling_med = temp.rolling(30, min_periods=3, center=True).median()
+        ti2 = temp.interpolate(method='linear')
+        temp[long_mask] = ti2.clip(lower=rolling_med*0.85, upper=rolling_med*1.15)[long_mask]
+
+    df[col] = temp
+    outlier_mask = df[col].pct_change().abs() > 0.4
+    if outlier_mask.any():
+        df.loc[outlier_mask, col] = np.nan
+        df[col] = df[col].interpolate(method='linear', limit=5)
+    return df
+
+def recommend_models(quality_info):
+    models = ['lightgbm', 'xgboost']
+    if quality_info['missing_pct'] < 30 and quality_info['max_gap_days'] < 14:
+        models.append('prophet')
+    if quality_info['cv'] < 5 and quality_info['missing_pct'] < 20:
+        models.append('arima')
+    if quality_info['total_records'] >= 90 and quality_info['missing_pct'] < 25:
+        models.append('sarima')
+    return models
+
+# ══════════════════════════════════════════════════════════════════════════════
+# 2. FEATURE ENGINEERING
+# ══════════════════════════════════════════════════════════════════════════════
+
+def create_features(df, target_col, other_cols=None):
+    df = df.copy()
+    df['year']          = df['date'].dt.year
+    df['month']         = df['date'].dt.month
+    df['day']           = df['date'].dt.day
+    df['day_of_week']   = df['date'].dt.dayofweek
+    df['day_of_year']   = df['date'].dt.dayofyear
+    df['week_of_year']  = df['date'].dt.isocalendar().week.astype(int)
+    df['quarter']       = df['date'].dt.quarter
+    df['is_weekend']    = (df['day_of_week'] >= 5).astype(int)
+    df['is_month_start']= df['date'].dt.is_month_start.astype(int)
+    df['is_month_end']  = df['date'].dt.is_month_end.astype(int)
+    for lag in [1, 2, 3, 7, 14, 30]:
+        df[f'price_lag_{lag}'] = df[target_col].shift(lag)
+    for w in [7, 14, 30]:
+        df[f'price_rolling_mean_{w}'] = df[target_col].rolling(w).mean()
+        df[f'price_rolling_std_{w}']  = df[target_col].rolling(w).std()
+        df[f'price_rolling_min_{w}']  = df[target_col].rolling(w).min()
+        df[f'price_rolling_max_{w}']  = df[target_col].rolling(w).max()
+    df['price_diff_1']       = df[target_col].diff(1)
+    df['price_diff_7']       = df[target_col].diff(7)
+    df['price_pct_change_1'] = df[target_col].pct_change(1) * 100
+    df['price_pct_change_7'] = df[target_col].pct_change(7) * 100
+    df['sma_7']    = df[target_col].rolling(7).mean()
+    df['sma_30']   = df[target_col].rolling(30).mean()
+    df['sma_diff'] = df['sma_7'] - df['sma_30']
+    df['volatility_7']  = df[target_col].rolling(7).std()
+    df['volatility_30'] = df[target_col].rolling(30).std()
+    if other_cols:
+        for i, oc in enumerate(other_cols):
+            if oc in df.columns:
+                df[f'other_price_{i}']  = df[oc]
+                df[f'price_spread_{i}'] = df[target_col] - df[oc]
+                df[f'price_ratio_{i}']  = df[target_col] / (df[oc] + 1e-6)
+    return df
+
+# ══════════════════════════════════════════════════════════════════════════════
+# 3. TRAINING
+# ══════════════════════════════════════════════════════════════════════════════
+
+def evaluate_model(y_true, y_pred, model_name=""):
+    rmse = float(np.sqrt(mean_squared_error(y_true, y_pred)))
+    return {
+        'model'    : model_name,
+        'rmse'     : rmse,
+        'mae'      : float(mean_absolute_error(y_true, y_pred)),
+        'r2'       : float(r2_score(y_true, y_pred)),
+        'mape'     : float(np.mean(np.abs((y_true - y_pred) / (np.abs(y_true)+1e-6))) * 100),
+        'median_ae': float(np.median(np.abs(y_true - y_pred))),
+    }
+
+def train_lightgbm(X_train, y_train, X_test, y_test):
+    m = lgb.LGBMRegressor(
+        objective='regression', metric='rmse', num_leaves=31,
+        max_depth=7, learning_rate=0.05, n_estimators=300,
+        min_child_samples=20, subsample=0.8, colsample_bytree=0.8,
+        reg_alpha=0.1, reg_lambda=0.1, verbose=-1,
+        random_state=42, force_col_wise=True,
+    )
+    m.fit(X_train, y_train, eval_set=[(X_test, y_test)],
+          eval_metric='rmse', callbacks=[lgb.early_stopping(50, verbose=False)])
+    return m, evaluate_model(y_test, m.predict(X_test), 'LightGBM')
+
+def train_xgboost(X_train, y_train, X_test, y_test):
+    m = xgb.XGBRegressor(
+        objective='reg:squarederror', n_estimators=300, max_depth=6,
+        learning_rate=0.05, subsample=0.8, colsample_bytree=0.8,
+        reg_alpha=0.1, reg_lambda=0.1, random_state=42, verbosity=0,
+    )
+    m.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=False)
+    return m, evaluate_model(y_test, m.predict(X_test), 'XGBoost')
+
+def train_prophet(dates_train, y_train, dates_test, y_test):
+    """Train Prophet with robust error handling and data validation."""
+    try:
+        from prophet import Prophet
+    except ImportError:
+        raise RuntimeError("Prophet not installed")
+    # Ensure clean datetime + float data
+    ds_train = pd.to_datetime(dates_train).reset_index(drop=True)
+    y_tr = y_train.reset_index(drop=True).astype(float)
+    ds_test = pd.to_datetime(dates_test).reset_index(drop=True)
+    y_te = y_test.reset_index(drop=True).astype(float)
+    # Remove NaN rows
+    mask = y_tr.notna()
+    ds_train = ds_train[mask]
+    y_tr = y_tr[mask]
+    if len(y_tr) < 30:
+        raise ValueError("Prophet: data training terlalu sedikit setelah cleaning")
+    df_train = pd.DataFrame({'ds': ds_train.values, 'y': y_tr.values})
+    df_test = pd.DataFrame({'ds': ds_test.values})
+    m = Prophet(
+        yearly_seasonality=True, weekly_seasonality=True,
+        daily_seasonality=False, changepoint_prior_scale=0.05,
+    )
+    
+    # Matikan log dengan cara yang lebih aman
+    logging.getLogger('prophet').setLevel(logging.ERROR)
+    logging.getLogger('cmdstanpy').setLevel(logging.ERROR)
+    
+    m.fit(df_train)
+    fc = m.predict(df_test)
+    pred = np.clip(fc['yhat'].values, 0, None)
+    return m, evaluate_model(y_te.values, pred, 'Prophet')
+
+def train_arima(y_train, y_test):
+    m = ARIMA(y_train.values, order=(5, 1, 0)).fit()
+    fc = np.clip(m.forecast(steps=len(y_test)), 0, None)
+    return m, evaluate_model(y_test.values, fc, 'ARIMA')
+
+def train_sarima(y_train, y_test):
+    """Train SARIMA with seasonal_order for weekly patterns."""
+    m = SARIMAX(
+        y_train.values, order=(1, 1, 1), seasonal_order=(1, 1, 1, 7),
+        enforce_stationarity=False, enforce_invertibility=False,
+    ).fit(disp=False, maxiter=200)
+    fc = np.clip(m.forecast(steps=len(y_test)), 0, None)
+    return m, evaluate_model(y_test.values, fc, 'SARIMA')
+
+def forecast_sarima(model, n_days=7):
+    """Forecast n_days ahead using fitted SARIMA model."""
+    fc = np.clip(model.forecast(steps=n_days), 0, None)
+    last_date = pd.Timestamp.now().normalize()
+    return [
+        {'tanggal': str((last_date + pd.Timedelta(days=i+1)).date()),
+         'harga_prediksi': round(float(fc[i]), 2)}
+        for i in range(n_days)
+    ]
+
+def tune_with_optuna(model_name, X_train, y_train, n_trials=50):
+    def objective(trial):
+        if model_name == 'LightGBM':
+            params = {
+                'objective':'regression','metric':'rmse','verbose':-1,
+                'random_state':42,'force_col_wise':True,
+                'num_leaves'       : trial.suggest_int('num_leaves', 20, 100),
+                'max_depth'        : trial.suggest_int('max_depth', 4, 12),
+                'learning_rate'    : trial.suggest_float('learning_rate', 0.01, 0.3),
+                'n_estimators'     : trial.suggest_int('n_estimators', 100, 500),
+                'min_child_samples': trial.suggest_int('min_child_samples', 5, 50),
+                'subsample'        : trial.suggest_float('subsample', 0.5, 1.0),
+                'colsample_bytree' : trial.suggest_float('colsample_bytree', 0.5, 1.0),
+                'reg_alpha'        : trial.suggest_float('reg_alpha', 0.0, 1.0),
+                'reg_lambda'       : trial.suggest_float('reg_lambda', 0.0, 1.0),
+            }
+            Cls = lgb.LGBMRegressor
+        else:
+            params = {
+                'objective':'reg:squarederror','random_state':42,'verbosity':0,
+                'n_estimators'    : trial.suggest_int('n_estimators', 100, 500),
+                'max_depth'       : trial.suggest_int('max_depth', 3, 10),
+                'learning_rate'   : trial.suggest_float('learning_rate', 0.01, 0.3),
+                'subsample'       : trial.suggest_float('subsample', 0.5, 1.0),
+                'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),
+                'reg_alpha'       : trial.suggest_float('reg_alpha', 0.0, 1.0),
+                'reg_lambda'      : trial.suggest_float('reg_lambda', 0.0, 1.0),
+            }
+            Cls = xgb.XGBRegressor
+        scores = []
+        for tr, val in TimeSeriesSplit(n_splits=5).split(X_train):
+            m = Cls(**params)
+            m.fit(X_train.iloc[tr], y_train.iloc[tr])
+            scores.append(np.sqrt(mean_squared_error(y_train.iloc[val], m.predict(X_train.iloc[val]))))
+        return np.mean(scores)
+
+    study = optuna.create_study(direction='minimize')
+    study.optimize(objective, n_trials=n_trials)
+    return study.best_params
+
+# ══════════════════════════════════════════════════════════════════════════════
+# 4. FORECASTING
+# ══════════════════════════════════════════════════════════════════════════════
+
+def forecast_tree(model, df_hist, target_col, other_cols, feature_cols, n_days=7):
+    df_temp   = df_hist.copy()
+    preds     = []
+    last_date = df_temp['date'].max()
+    for i in range(1, n_days + 1):
+        next_date = last_date + pd.Timedelta(days=i)
+        new_row   = {'date': next_date, target_col: np.nan}
+        for oc in (other_cols or []):
+            if oc in df_temp.columns:
+                lo = df_temp[oc].dropna()
+                new_row[oc] = float(lo.iloc[-1]) if len(lo) else np.nan
+        df_temp = pd.concat([df_temp, pd.DataFrame([new_row])], ignore_index=True)
+        df_feat  = create_features(df_temp, target_col, other_cols)
+        last_row = df_feat.iloc[[-1]].copy()
+        for col in feature_cols:
+            if col not in last_row.columns:
+                last_row[col] = 0.0
+        pred = max(0, float(model.predict(last_row[feature_cols])[0]))
+        preds.append({'tanggal': str(next_date.date()), 'harga_prediksi': round(pred, 2)})
+        df_temp.loc[df_temp['date'] == next_date, target_col] = pred
+    return preds
+
+def forecast_arima(model, n_days=7):
+    """Forecast n_days ahead using fitted ARIMA model."""
+    fc = np.clip(model.forecast(steps=n_days), 0, None)
+    last_date = pd.Timestamp.now().normalize()
+    return [
+        {'tanggal': str((last_date + pd.Timedelta(days=i+1)).date()),
+         'harga_prediksi': round(float(fc[i]), 2)}
+        for i in range(n_days)
+    ]
+
+def forecast_prophet(model, last_date, n_days=7):
+    future = pd.DataFrame({'ds': pd.date_range(start=last_date + pd.Timedelta(days=1), periods=n_days)})
+    fc = model.predict(future)
+    return [{'tanggal': str(r.ds.date()), 'harga_prediksi': round(max(0, r.yhat), 2)}
+            for _, r in fc.iterrows()]
+
+# ══════════════════════════════════════════════════════════════════════════════
+# 5. LABEL GENERATOR (untuk frontend)
+# ═════════════════════════════════════��════════════════════════════════════════
+
+def get_tren(preds):
+    pct = (preds[-1]['harga_prediksi'] - preds[0]['harga_prediksi']) / preds[0]['harga_prediksi'] * 100
+    if pct > 2:     return 'naik_tajam',       pct
+    elif pct > 0.5: return 'naik_terkendali',  pct
+    elif pct < -2:  return 'turun_tajam',       pct
+    elif pct < -0.5: return 'turun_terkendali', pct
+    else:           return 'stabil',            pct
+
+def get_labels(mape, rmse, mean_price, cv, preds, komoditas_nama):
+    harga_vals      = [p['harga_prediksi'] for p in preds]
+    tren, pct       = get_tren(preds)
+    confidence      = round(max(0, min(100, 100 - mape * 2)), 2)
+    stabilitas      = ('optimal' if mape < 2 else 'baik' if mape < 5
+                       else 'cukup' if mape < 10 else 'perlu_retrain')
+    mape_label      = ('Sangat Rendah' if mape < 5 else 'Rendah' if mape < 10
+                       else 'Sedang' if mape < 15 else 'Tinggi' if mape < 25 else 'Sangat Tinggi')
+    rmse_pct        = rmse / mean_price * 100
+    rmse_label      = ('Presisi Sangat Tinggi' if rmse_pct < 1 else 'Presisi Tinggi' if rmse_pct < 2
+                       else 'Presisi Sedang' if rmse_pct < 5 else 'Presisi Rendah')
+    cl_label        = ('Sangat Tinggi' if confidence >= 90 else 'Tinggi' if confidence >= 80
+                       else 'Sedang' if confidence >= 65 else 'Rendah')
+    vol_label       = ('Rendah' if cv < 2 else 'Sedang' if cv < 5 else 'Tinggi')
+
+    if mape < 10:
+        status = {'judul': 'Akurasi Terverifikasi',
+                  'deskripsi': f'Prediksi divalidasi, deviasi rata-rata di bawah {mape:.1f}%.'}
+    elif mape < 20:
+        status = {'judul': 'Akurasi Cukup',
+                  'deskripsi': f'Deviasi rata-rata {mape:.1f}%. Gunakan sebagai referensi.'}
+    else:
+        status = {'judul': 'Perlu Perhatian',
+                  'deskripsi': f'Deviasi {mape:.1f}%. Disarankan retraining.'}
+
+    tren_map = {
+        'stabil'          : ('positif', 'check-circle',   f'Harga {komoditas_nama} diprediksi stabil 7 hari ke depan.'),
+        'naik_terkendali' : ('negatif', 'trending-up',    f'Harga {komoditas_nama} diprediksi naik {abs(pct):.1f}%.'),
+        'naik_tajam'      : ('negatif', 'alert-triangle', f'Harga {komoditas_nama} diprediksi naik tajam {abs(pct):.1f}%.'),
+        'turun_terkendali': ('positif', 'trending-down',  f'Harga {komoditas_nama} diprediksi turun {abs(pct):.1f}%.'),
+        'turun_tajam'     : ('netral',  'alert-triangle', f'Harga {komoditas_nama} diprediksi turun tajam {abs(pct):.1f}%.'),
+    }
+    t1, ikon1, k1 = tren_map.get(tren, ('netral', 'info', f'Tren: {tren}'))
+    peak   = max(preds, key=lambda x: x['harga_prediksi'])
+    trough = min(preds, key=lambda x: x['harga_prediksi'])
+
+    insights = [
+        {'tipe': t1, 'ikon': ikon1, 'konten': k1, 'urutan': 1},
+        {
+            'tipe'  : 'positif' if cv < 2 else ('netral' if cv < 5 else 'negatif'),
+            'ikon'  : 'check-circle' if cv < 2 else 'info',
+            'konten': ('Pasokan terpantau mencukupi.' if cv < 2
+                       else f'Volatilitas sedang (CV={cv:.1f}%).' if cv < 5
+                       else f'Harga sangat fluktuatif (CV={cv:.1f}%).'),
+            'urutan': 2,
+        },
+        {
+            'tipe'  : 'netral',
+            'ikon'  : 'calendar',
+            'konten': (f"Harga tertinggi Rp {peak['harga_prediksi']:,.0f}, "
+                       f"terendah Rp {trough['harga_prediksi']:,.0f} dalam 7 hari."),
+            'urutan': 3,
+        },
+    ]
+
+    return {
+        'tren'            : tren,
+        'pct_change'      : round(pct, 2),
+        'confidence_level': confidence,
+        'confidence_label': cl_label,
+        'stabilitas'      : stabilitas,
+        'mape_label'      : mape_label,
+        'rmse_label'      : rmse_label,
+        'volatilitas_label': vol_label,
+        'status_analisis' : status,
+        'insights'        : insights,
+        'harga_min'       : min(harga_vals),
+        'harga_max'       : max(harga_vals),
+    }
+
+# ══════════════════════════════════════════════════════════════════════════════
+# 6. MAIN PIPELINE — dipanggil oleh API
+# ══════════════════════════════════════════════════════════════════════════════
+
+def run_pipeline(komoditas_id: int, komoditas_nama: str, df: pd.DataFrame,
+                 target_market: str, pasar_id: int, n_trials: int = 50,
+                 forecast_days: int = 7, log_cb=None):
+    """
+    Full pipeline: cleaning → training → tuning → forecasting → labeling → save.
+    log_cb: callback function(msg: str) untuk streaming log ke frontend.
+    """
+    def log(msg):
+        if log_cb: log_cb(msg)
+
+    other_cols = [c for c in df.columns if c not in ['date', target_market]
+                  and not c.endswith('_was_missing')]
+
+    # ── Cleaning ──
+    log("🧹 Cleaning data...")
+    all_cols = [target_market] + other_cols
+    quality_reports = {}
+    for col in all_cols:
+        if col in df.columns:
+            q = analyze_data_quality(df, col)
+            quality_reports[col] = q
+            df = clean_price_data(df, col, q)
+
+    models_to_run = recommend_models(quality_reports.get(target_market, {}))
+    log(f"🤖 Model yang akan dijalankan: {models_to_run}")
+
+    # ── Prepare dataset ──
+    exclude = ['date'] + [c for c in df.columns if c.endswith('_was_missing')]
+    df_feat = create_features(df, target_market, other_cols)
+    df_clean = df_feat.dropna(subset=[target_market]).reset_index(drop=True)
+    feature_cols = [c for c in df_clean.columns if c not in exclude + all_cols]
+
+    X = df_clean[feature_cols]
+    y = df_clean[target_market]
+    dates = df_clean['date']
+    split = int(len(df_clean) * 0.7)
+    X_train, X_test = X.iloc[:split], X.iloc[split:]
+    y_train, y_test = y.iloc[:split], y.iloc[split:]
+    mean_price = float(y.mean())
+
+    # ── Train ──
+    all_results = {}
+    lgb_model = xgb_model = prophet_model = None
+
+    if 'lightgbm' in models_to_run:
+        log("Training LightGBM...")
+        lgb_model, res = train_lightgbm(X_train, y_train, X_test, y_test)
+        all_results['LightGBM'] = {**res, 'obj': lgb_model, 'type': 'tree'}
+
+    if 'xgboost' in models_to_run:
+        log("Training XGBoost...")
+        xgb_model, res = train_xgboost(X_train, y_train, X_test, y_test)
+        all_results['XGBoost'] = {**res, 'obj': xgb_model, 'type': 'tree'}
+
+    if 'prophet' in models_to_run:
+        log("Training Prophet...")
+        try:
+            prophet_model, res = train_prophet(dates.iloc[:split], y_train, dates.iloc[split:], y_test)
+            all_results['Prophet'] = {**res, 'obj': prophet_model, 'type': 'prophet'}
+        except Exception as e:
+            log(f"⚠️ Prophet gagal: {e}")
+
+    if 'arima' in models_to_run:
+        log("Training ARIMA...")
+        try:
+            arima_model, res = train_arima(y_train, y_test)
+            all_results['ARIMA'] = {**res, 'obj': arima_model, 'type': 'arima'}
+        except Exception as e:
+            log(f"⚠️ ARIMA gagal: {e}")
+
+    if 'sarima' in models_to_run:
+        log("Training SARIMA...")
+        try:
+            sarima_model, res = train_sarima(y_train, y_test)
+            all_results['SARIMA'] = {**res, 'obj': sarima_model, 'type': 'sarima'}
+        except Exception as e:
+            log(f"⚠️ SARIMA gagal: {e}")
+
+    best_name = min(all_results, key=lambda k: all_results[k]['rmse'])
+    best      = all_results[best_name]
+    log(f"🏆 Best model: {best_name} (RMSE={best['rmse']:.2f})")
+
+    # ── Tuning ──
+    final_model  = best['obj']
+    final_result = best
+    final_name   = best_name
+
+    if best['type'] == 'tree':
+        log(f"🔧 Tuning {best_name} dengan Optuna ({n_trials} trials)...")
+        best_params = tune_with_optuna(best_name, X_train, y_train, n_trials)
+        if best_name == 'LightGBM':
+            tuned = lgb.LGBMRegressor(**best_params, objective='regression',
+                                      verbose=-1, random_state=42, force_col_wise=True)
+        else:
+            tuned = xgb.XGBRegressor(**best_params, objective='reg:squarederror',
+                                     random_state=42, verbosity=0)
+        tuned.fit(X_train, y_train)
+        tuned_res = evaluate_model(y_test, tuned.predict(X_test), f"{best_name}_Tuned")
+        if tuned_res['rmse'] < best['rmse']:
+            final_model  = tuned
+            final_result = tuned_res
+            final_name   = f"{best_name}_Tuned"
+            log(f"✅ Tuned lebih baik, improvement={(best['rmse']-tuned_res['rmse'])/best['rmse']*100:.1f}%")
+
+    # ── Forecast ──
+    n_fc = forecast_days
+    log(f"📅 Forecasting {n_fc} hari...")
+    df_hist = df[['date', target_market] + [c for c in other_cols if c in df.columns]].copy()
+    f_type = final_result.get('type', '')
+    if f_type == 'prophet' or final_name == 'Prophet':
+        predictions = forecast_prophet(final_model, df_hist['date'].max(), n_fc)
+    elif f_type == 'arima':
+        predictions = forecast_arima(final_model, n_fc)
+    elif f_type == 'sarima':
+        predictions = forecast_sarima(final_model, n_fc)
+    else:
+        predictions = forecast_tree(final_model, df_hist, target_market, other_cols, feature_cols, n_fc)
+
+    # ── Labels ──
+    cv = quality_reports.get(target_market, {}).get('cv', 0)
+    labels = get_labels(
+        final_result['mape'], final_result['rmse'],
+        mean_price, cv, predictions, komoditas_nama
+    )
+
+    # ── Simpan model ──
+    prefix    = os.path.join(MODEL_DIR, f"model_{komoditas_id}_{pasar_id}")
+    model_path= f"{prefix}.pkl"
+    with open(model_path, 'wb') as f:
+        pickle.dump(final_model, f)
+    with open(f"{prefix}_features.json", 'w') as f:
+        json.dump(feature_cols, f)
+
+    metadata = {
+        'komoditas_id'    : komoditas_id,
+        'komoditas_nama'  : komoditas_nama,
+        'pasar_id'        : pasar_id,
+        'target_market'   : target_market,
+        'other_markets'   : other_cols,
+        'nama_model'      : final_name,
+        'versi'           : '1.0',
+        'file_path'       : model_path,
+        'mape'            : round(final_result['mape'], 4),
+        'rmse'            : round(final_result['rmse'], 4),
+        'mae'             : round(final_result['mae'], 4),
+        'r2_score'        : round(final_result['r2'], 4),
+        'confidence_level': labels['confidence_level'],
+        'stabilitas'      : labels['stabilitas'],
+        'status_validasi' : 'terverifikasi',
+        'catatan_validasi': labels['status_analisis']['deskripsi'],
+        'tanggal_training': date.today().isoformat(),
+        'tanggal_evaluasi': date.today().isoformat(),
+        'deskripsi'       : f"Model {final_name} untuk prediksi harga {komoditas_nama}.",
+        'models_tried'    : list(all_results.keys()),
+        'data_quality'    : quality_reports.get(target_market, {}),
+    }
+    with open(f"{prefix}_metadata.json", 'w') as f:
+        json.dump(metadata, f, indent=2)
+
+    log(f"✅ Pipeline selesai! MAPE={final_result['mape']:.2f}%")
+
+    return {
+        'predictions'    : predictions,
+        'labels'         : labels,
+        'metadata'       : metadata,
+        'model_comparison': {k: {
+            'rmse': round(v['rmse'], 4),
+            'mape': round(v['mape'], 4),
+            'mae': round(v['mae'], 4),
+            'r2': round(v['r2'], 4)
+        } for k, v in all_results.items()},
+    }

backend/app/scheduler.py

@@ -1,16 +1,29 @@
+# scheduler.py — Multiple-schedule Scraping and Forecasting Manager
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.triggers.cron import CronTrigger
 from app.scraper import scraping_hari_ini, upload_to_mysql
 from app.database import get_db_engine
 from app.config import get_settings
-from app.logger import get_app_logger
+from app.logger import get_app_logger, get_forecast_logger
 from datetime import datetime
 import uuid
-from pytz import timezone
 
-logger = get_app_logger("scheduler")
+try:
+    from pytz import timezone as pytz_timezone
+    wib_tz = pytz_timezone("Asia/Jakarta")
+except ImportError:
+    import zoneinfo
+    wib_tz = zoneinfo.ZoneInfo("Asia/Jakarta")
+
+# Loggers
+scraper_log = get_app_logger("scheduler")
+forecast_log = get_forecast_logger("scheduler")
+
 settings = get_settings()
-wib_tz = timezone("Asia/Jakarta")
+
+# ══════════════════════════════════════════════════════════════════════════════
+# 1. SCRAPER SCHEDULER
+# ══════════════════════════════════════════════════════════════════════════════
 
 class ScraperScheduler:
     def __init__(self):
@@ -21,7 +34,7 @@ class ScraperScheduler:
         self.schedules = {}  # id -> {"cron": str, "label": str}
         
     def run_scraping(self):
-        logger.info("⏰ Menjalankan tugas scraping terjadwal otomatis...")
+        scraper_log.info("⏰ Menjalankan tugas scraping terjadwal otomatis...")
         try:
             df = scraping_hari_ini()
             engine = get_db_engine()
@@ -32,21 +45,32 @@ class ScraperScheduler:
                 "stats": stats, 
                 "timestamp": self.last_run.isoformat()
             }
-            logger.info("✅ Tugas scraping otomatis selesai dijalankan.")
+            scraper_log.info("✅ Tugas scraping otomatis selesai dijalankan.")
         except Exception as e:
             self.last_result = {
                 "success": False, 
                 "error": str(e), 
                 "timestamp": datetime.now().isoformat()
             }
-            logger.error(f"❌ Tugas scraping otomatis gagal: {e}")
+            scraper_log.error(f"❌ Tugas scraping otomatis gagal: {e}")
     
     def start_all(self):
         if not self.is_running:
+            self.scheduler = BackgroundScheduler(timezone=wib_tz)
             self.scheduler.start()
             self.is_running = True
-            logger.info("▶️ Mesin Scheduler utama telah diaktifkan.")
+            scraper_log.info("▶️ Mesin Scheduler Scraper telah diaktifkan.")
             
+            # Re-register existing schedules
+            for jid, info in list(self.schedules.items()):
+                try:
+                    trigger = CronTrigger.from_crontab(info["cron"], timezone=wib_tz)
+                    self.scheduler.add_job(
+                        self.run_scraping, trigger, id=jid, replace_existing=True
+                    )
+                except Exception:
+                    pass
+
             # Load default schedules if empty
             if not self.schedules:
                 defaults = [s.strip() for s in settings.DEFAULT_SCHEDULES.split(",") if s.strip()]
@@ -55,15 +79,16 @@ class ScraperScheduler:
     
     def stop_all(self):
         if self.is_running:
-            self.scheduler.shutdown(wait=False)
-            self.scheduler = BackgroundScheduler() # reset
+            try:
+                self.scheduler.shutdown(wait=False)
+            except Exception:
+                pass
+            self.scheduler = BackgroundScheduler(timezone=wib_tz)
             self.is_running = False
             self.schedules.clear()
-            logger.info("⏹️ Mesin Scheduler utama telah dimatikan.")
+            scraper_log.info("⏹️ Mesin Scheduler Scraper telah dimatikan.")
             
     def add_schedule(self, cron_expression: str, label: str = "") -> str:
-        # Check if we already have this exact cron to avoid exact duplicates if desired,
-        # but let's allow unique IDs
         job_id = str(uuid.uuid4())[:8]
         try:
             trigger = CronTrigger.from_crontab(cron_expression, timezone=wib_tz)
@@ -76,35 +101,32 @@ class ScraperScheduler:
             if not label:
                 label = f"Jadwal {cron_expression}"
             self.schedules[job_id] = {"cron": cron_expression, "label": label}
-            logger.info(f"📅 Jadwal baru ditambahkan: {cron_expression} ({label})")
+            scraper_log.info(f"📅 Jadwal baru ditambahkan: {cron_expression} ({label})")
             
-            # Ensure scheduler is running if we add a job
             if not self.is_running:
                 self.scheduler.start()
                 self.is_running = True
-                logger.info("▶️ Mesin Scheduler otomatis menyala karena ada jadwal baru.")
+                scraper_log.info("▶️ Mesin Scheduler Scraper menyala karena ada jadwal baru.")
                 
             return job_id
         except Exception as e:
-            logger.error(f"❌ Gagal menambahkan jadwal cron '{cron_expression}': {e}")
+            scraper_log.error(f"❌ Gagal menambahkan jadwal cron '{cron_expression}': {e}")
             raise ValueError(f"Format Cron tidak valid: {e}")
 
     def remove_schedule(self, job_id: str):
         if job_id in self.schedules:
             try:
                 self.scheduler.remove_job(job_id)
-            except:
+            except Exception:
                 pass
             deleted = self.schedules.pop(job_id)
-            logger.info(f"🗑️ Jadwal dihapus: {deleted['cron']} ({deleted['label']})")
+            scraper_log.info(f"🗑️ Jadwal dihapus: {deleted['cron']} ({deleted['label']})")
         else:
             raise KeyError("ID Jadwal tidak ditemukan")
             
     def list_schedules(self):
         result = []
-        # Get active jobs from APScheduler to find next_run_time
         jobs_map = {job.id: job for job in self.scheduler.get_jobs()}
-        
         for jid, info in self.schedules.items():
             job_obj = jobs_map.get(jid)
             next_run = job_obj.next_run_time.isoformat() if (job_obj and job_obj.next_run_time) else None
@@ -117,8 +139,7 @@ class ScraperScheduler:
         return result
 
     def get_status(self):
-        # Let's find the absolute next run among all active jobs
-        jobs = self.scheduler.get_jobs()
+        jobs = self.scheduler.get_jobs() if self.is_running else []
         next_runs = [j.next_run_time for j in jobs if j.next_run_time]
         next_run_overall = min(next_runs).isoformat() if next_runs else None
         
@@ -130,4 +151,138 @@ class ScraperScheduler:
             "active_schedules_count": len(self.schedules)
         }
 
-scraper_scheduler = ScraperScheduler()
+# ══════════════════════════════════════════════════════════════════════════════
+# 2. FORECAST SCHEDULER
+# ══════════════════════════════════════════════════════════════════════════════
+
+class ForecastScheduler:
+    def __init__(self):
+        self.scheduler = BackgroundScheduler(timezone=wib_tz)
+        self.is_running = False
+        self.last_run = None
+        self.last_result = None
+        self.schedules = {}  # id -> {"cron": str, "label": str}
+        self._forecast_job_fn = None
+
+    def set_forecast_job(self, fn):
+        self._forecast_job_fn = fn
+
+    def _run_job(self):
+        forecast_log.info("⏰ Menjalankan forecast terjadwal otomatis...")
+        try:
+            if self._forecast_job_fn:
+                self._forecast_job_fn()
+            self.last_run = datetime.now()
+            self.last_result = {
+                "success": True,
+                "timestamp": self.last_run.isoformat(),
+            }
+            forecast_log.info("✅ Forecast terjadwal selesai.")
+        except Exception as e:
+            self.last_result = {
+                "success": False,
+                "error": str(e),
+                "timestamp": datetime.now().isoformat(),
+            }
+            forecast_log.error(f"❌ Forecast terjadwal gagal: {e}")
+
+    def start_all(self, default_schedules: str = ""):
+        if not self.is_running:
+            self.scheduler = BackgroundScheduler(timezone=wib_tz)
+            self.scheduler.start()
+            self.is_running = True
+            forecast_log.info("▶️ Mesin Scheduler Forecast telah diaktifkan.")
+
+            # Re-register existing schedules
+            for jid, info in list(self.schedules.items()):
+                try:
+                    trigger = CronTrigger.from_crontab(info["cron"], timezone=wib_tz)
+                    self.scheduler.add_job(
+                        self._run_job, trigger, id=jid, replace_existing=True
+                    )
+                except Exception:
+                    pass
+
+            # Load defaults if empty
+            if not self.schedules and default_schedules:
+                for cron in [s.strip() for s in default_schedules.split(",") if s.strip()]:
+                    self.add_schedule(cron, label="Jadwal Default")
+
+    def stop_all(self):
+        if self.is_running:
+            try:
+                self.scheduler.shutdown(wait=False)
+            except Exception:
+                pass
+            self.scheduler = BackgroundScheduler(timezone=wib_tz)
+            self.is_running = False
+            self.schedules.clear()
+            forecast_log.info("⏹️ Mesin Scheduler Forecast telah dimatikan.")
+
+    def add_schedule(self, cron_expression: str, label: str = "") -> str:
+        job_id = str(uuid.uuid4())[:8]
+        try:
+            trigger = CronTrigger.from_crontab(cron_expression, timezone=wib_tz)
+            self.scheduler.add_job(
+                self._run_job, trigger, id=job_id, replace_existing=True
+            )
+            if not label:
+                label = f"Jadwal {cron_expression}"
+            self.schedules[job_id] = {"cron": cron_expression, "label": label}
+            forecast_log.info(f"📅 Jadwal baru ditambahkan: {cron_expression} ({label})")
+
+            if not self.is_running:
+                self.scheduler.start()
+                self.is_running = True
+                forecast_log.info("▶️ Scheduler otomatis menyala karena ada jadwal baru.")
+
+            return job_id
+        except Exception as e:
+            forecast_log.error(f"❌ Gagal menambahkan jadwal cron '{cron_expression}': {e}")
+            raise ValueError(f"Format Cron tidak valid: {e}")
+
+    def remove_schedule(self, job_id: str):
+        if job_id in self.schedules:
+            try:
+                self.scheduler.remove_job(job_id)
+            except Exception:
+                pass
+            deleted = self.schedules.pop(job_id)
+            forecast_log.info(f"🗑️ Jadwal dihapus: {deleted['cron']} ({deleted['label']})")
+        else:
+            raise KeyError("ID Jadwal tidak ditemukan")
+
+    def list_schedules(self):
+        result = []
+        jobs_map = {job.id: job for job in self.scheduler.get_jobs()}
+        for jid, info in self.schedules.items():
+            job_obj = jobs_map.get(jid)
+            next_run = (
+                job_obj.next_run_time.isoformat()
+                if (job_obj and job_obj.next_run_time)
+                else None
+            )
+            result.append({
+                "id": jid,
+                "cron_expression": info["cron"],
+                "label": info["label"],
+                "next_run": next_run,
+            })
+        return result
+
+    def get_status(self):
+        jobs = self.scheduler.get_jobs() if self.is_running else []
+        next_runs = [j.next_run_time for j in jobs if j.next_run_time]
+        next_run_overall = min(next_runs).isoformat() if next_runs else None
+
+        return {
+            "is_running": self.is_running,
+            "last_run": self.last_run.isoformat() if self.last_run else None,
+            "last_result": self.last_result,
+            "next_run": next_run_overall,
+            "active_schedules_count": len(self.schedules),
+        }
+
+# Singleton instances
+scraper_scheduler = ScraperScheduler()
+forecast_scheduler = ForecastScheduler()

backend/requirements.txt

@@ -10,4 +10,11 @@ apscheduler==3.10.4
 pydantic==2.5.3
 python-multipart==0.0.6
 pydantic-settings==2.1.0
-aiofiles==23.2.1
+aiofiles==23.2.1
+scikit-learn==1.4.2
+lightgbm==4.3.0
+xgboost==2.0.3
+prophet==1.1.5
+statsmodels==0.14.2
+optuna==3.6.1
+pytz==2024.1