Baseline commit: full project state before audit fixes

.env.example

@@ -15,3 +15,8 @@ CACHE_MAX_AGE_HOURS=24
 
 # Logging
 LOG_LEVEL=INFO
+
+# Cron ($0: internal scheduler + optional free HF Space Scheduler UI)
+CRON_SECRET=generate-a-long-random-string
+ENABLE_INTERNAL_CRON=true
+DAILY_FORECAST_LIMIT_PER_COUNTRY=50

.gitignore

@@ -1,30 +1,7 @@
-# Python
-__pycache__/
-*.py[cod]
-*$py.class
-*.so
-*.egg-info/
-*.egg
 venv/
-env/
-ENV/
-
-# Environment
+__pycache__/
+*.pyc
 .env
-.env.local
-
-# Logs
-*.log
-
-# IDE
-.vscode/
-.idea/
-*.swp
-
-# OS
-.DS_Store
-Thumbs.db
-
-# Database
+data/
 *.db
-*.sqlite
+.pytest_cache/

Dockerfile

@@ -2,21 +2,20 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# Install system dependencies
 RUN apt-get update && apt-get install -y \
     gcc \
     g++ \
+    libcairo2 \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy requirements and install Python dependencies
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Copy application code
 COPY app/ ./app/
+COPY scripts/ ./scripts/
 
-# Expose port
+ENV API_PORT=7860
 EXPOSE 7860
 
-# Run the application
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
+# HuggingFace Spaces expect port 7860
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -52,14 +52,25 @@ uvicorn app.main:app --reload --port 8000
 # API docs available at http://localhost:8000/docs
 ```
 
-### Deployment to HuggingFace Spaces
+### Deployment to HuggingFace Spaces ($0 tier)
 
-1. Create a new Space on HuggingFace (Docker type)
-2. Push this directory to the Space repository
-3. Set environment variables in Space settings:
-   - `TURSO_URL`
-   - `TURSO_TOKEN`
-4. Space will automatically build and deploy
+**Never enable paid HF Jobs or GPU hardware** — use free `cpu-basic` only.
+
+1. Create a **Docker** Space on [huggingface.co/spaces](https://huggingface.co/spaces)
+2. Push `backend/` to the Space (`.\deploy-hf.ps1` from repo root)
+3. Run `python scripts/configure_hf_space.py` — sets secrets + `ENABLE_INTERNAL_CRON=true`
+4. **Optional (free):** Space → **Scheduler** — two HTTP jobs to wake the Space after idle sleep (see `DEPLOYMENT.md` §4). Internal cron already runs jobs while the Space is up.
+5. Set Cloudflare `PUBLIC_API_URL` to `https://<user>-<space>.hf.space`
+
+**Cron endpoints** (all require `X-Cron-Secret` header):
+
+| Endpoint | Purpose |
+|----------|---------|
+| `POST /api/v1/cron/daily` | Metadata sync + full forecast pipeline (recommended) |
+| `POST /api/v1/cron/forecasts` | Forecasts only |
+| `POST /api/v1/cron/metadata/daily` | Stock metadata / status only |
+| `POST /api/v1/cron/metadata/weekly` | Full directory refresh (Sundays) |
+| `GET /api/v1/cron/health` | Verify routes are live |
 
 ## API Endpoints
 
@@ -128,6 +139,8 @@ app/
 | `TIMESFM_MODEL` | Model ID (default: google/timesfm-2.5-200m-pytorch) | No |
 | `DEVICE` | Compute device (cpu/cuda) | No |
 | `CACHE_MAX_AGE_HOURS` | Cache TTL (default: 24) | No |
+| `CRON_SECRET` | Secret for HF Scheduler cron HTTP calls | Yes (production) |
+| `DAILY_FORECAST_LIMIT_PER_COUNTRY` | Max stocks per country per daily run | No |
 
 ## Performance
 

app/cron.py

@@ -0,0 +1,84 @@
+"""
+Secured cron endpoints for HuggingFace Space Scheduler (replaces GitHub Actions).
+Configure in Space → Settings → Scheduler → add job pointing at these URLs.
+"""
+
+import os
+from fastapi import APIRouter, Header, HTTPException, BackgroundTasks
+from dotenv import load_dotenv
+
+from app import jobs
+
+load_dotenv()
+
+router = APIRouter(prefix="/api/v1/cron", tags=["cron"])
+
+
+def _cron_secret() -> str:
+    return os.getenv("CRON_SECRET", "")
+
+
+def _verify_secret(x_cron_secret: str | None):
+    secret = _cron_secret()
+    if not secret:
+        raise HTTPException(
+            status_code=503,
+            detail="CRON_SECRET not set on server. Add it in HuggingFace Space secrets.",
+        )
+    if x_cron_secret != secret:
+        raise HTTPException(status_code=401, detail="Invalid cron secret")
+
+
+@router.get("/health")
+async def cron_health():
+    """Public check that cron routes are mounted (no secret required)."""
+    return {
+        "status": "ok",
+        "cron_configured": bool(_cron_secret()),
+        "hint": "HF Scheduler should POST/GET /api/v1/cron/daily with header X-Cron-Secret",
+    }
+
+
+@router.post("/daily")
+async def cron_daily(
+    background_tasks: BackgroundTasks,
+    x_cron_secret: str | None = Header(default=None),
+):
+    """
+    Main daily automation (run at 21:00 UTC weekdays via HF Scheduler).
+    Metadata sync + forecast generation. Runs in background; returns immediately.
+    """
+    _verify_secret(x_cron_secret)
+    background_tasks.add_task(jobs.run_full_daily_job)
+    return {"accepted": True, "job": "full_daily", "message": "Running in background"}
+
+
+@router.post("/forecasts")
+async def cron_forecasts(
+    background_tasks: BackgroundTasks,
+    x_cron_secret: str | None = Header(default=None),
+):
+    """Forecast pipeline only."""
+    _verify_secret(x_cron_secret)
+    background_tasks.add_task(jobs.run_daily_forecasts)
+    return {"accepted": True, "job": "daily_forecasts"}
+
+
+@router.post("/metadata/daily")
+async def cron_metadata_daily(
+    background_tasks: BackgroundTasks,
+    x_cron_secret: str | None = Header(default=None),
+):
+    _verify_secret(x_cron_secret)
+    background_tasks.add_task(jobs.run_metadata_daily)
+    return {"accepted": True, "job": "metadata_daily"}
+
+
+@router.post("/metadata/weekly")
+async def cron_metadata_weekly(
+    background_tasks: BackgroundTasks,
+    x_cron_secret: str | None = Header(default=None),
+):
+    _verify_secret(x_cron_secret)
+    background_tasks.add_task(jobs.run_metadata_weekly)
+    return {"accepted": True, "job": "metadata_weekly"}

app/internal_cron.py

@@ -0,0 +1,111 @@
+"""
+In-process cron for HuggingFace Spaces (free tier).
+Runs when the Space is awake; complements HF Settings → Scheduler HTTP pings.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from datetime import datetime, timedelta, timezone
+
+from app import jobs
+
+logger = logging.getLogger(__name__)
+
+DAILY_CRON = os.getenv("INTERNAL_DAILY_CRON", "0 21 * * 1-5")
+WEEKLY_CRON = os.getenv("INTERNAL_WEEKLY_CRON", "0 2 * * 0")
+
+
+def _enabled() -> bool:
+    return os.getenv("ENABLE_INTERNAL_CRON", "").lower() in ("1", "true", "yes")
+
+
+def _parse_cron_field(field: str, min_v: int, max_v: int) -> set[int] | None:
+    """Minimal cron field parser: *, N, N-M, */S, lists."""
+    if field == "*":
+        return None
+    out: set[int] = set()
+    for part in field.split(","):
+        if "/" in part:
+            base, step = part.split("/", 1)
+            step_n = int(step)
+            if base == "*":
+                vals = range(min_v, max_v + 1, step_n)
+            else:
+                start, end = (int(x) for x in base.split("-", 1)) if "-" in base else (int(base), max_v)
+                vals = range(start, end + 1, step_n)
+            out.update(vals)
+        elif "-" in part:
+            a, b = (int(x) for x in part.split("-", 1))
+            out.update(range(a, b + 1))
+        else:
+            out.add(int(part))
+    return out
+
+
+def _cron_matches(expr: str, dt: datetime) -> bool:
+    parts = expr.split()
+    if len(parts) != 5:
+        return False
+    minute, hour, dom, month, dow = parts
+    if _parse_cron_field(minute, 0, 59) is not None and dt.minute not in _parse_cron_field(minute, 0, 59):
+        return False
+    if _parse_cron_field(hour, 0, 23) is not None and dt.hour not in _parse_cron_field(hour, 0, 23):
+        return False
+    if _parse_cron_field(dom, 1, 31) is not None and dt.day not in _parse_cron_field(dom, 1, 31):
+        return False
+    if _parse_cron_field(month, 1, 12) is not None and dt.month not in _parse_cron_field(month, 1, 12):
+        return False
+    # Sunday=0 in standard cron
+    dow_set = _parse_cron_field(dow, 0, 6)
+    if dow_set is not None:
+        py_dow = (dt.weekday() + 1) % 7  # Mon=1..Sun=0
+        if py_dow not in dow_set:
+            return False
+    return True
+
+
+def _seconds_until_next(expr: str, now: datetime) -> float:
+    for i in range(366 * 24 * 60):
+        t = now + timedelta(minutes=i)
+        if _cron_matches(expr, t):
+            target = t.replace(second=0, microsecond=0)
+            if target <= now:
+                continue
+            return (target - now).total_seconds()
+    return 3600.0
+
+
+async def _run_loop(expr: str, job_name: str, coro_factory) -> None:
+    while True:
+        now = datetime.now(timezone.utc)
+        delay = _seconds_until_next(expr, now)
+        logger.info("Internal cron %s: next run in %.0fs (%s UTC)", job_name, delay, now.isoformat())
+        await asyncio.sleep(delay)
+        if not os.getenv("CRON_SECRET"):
+            logger.warning("CRON_SECRET unset; skipping %s", job_name)
+            continue
+        try:
+            logger.info("Internal cron starting %s", job_name)
+            await coro_factory()
+            logger.info("Internal cron finished %s", job_name)
+        except Exception as e:
+            logger.exception("Internal cron %s failed: %s", job_name, e)
+        await asyncio.sleep(60)
+
+
+async def start_internal_cron() -> list[asyncio.Task]:
+    if not _enabled():
+        logger.info("Internal cron disabled (set ENABLE_INTERNAL_CRON=true)")
+        return []
+    if not os.getenv("CRON_SECRET"):
+        logger.warning("CRON_SECRET unset; internal cron not started")
+        return []
+    tasks = [
+        asyncio.create_task(_run_loop(DAILY_CRON, "daily_forecasts", jobs.run_full_daily_job)),
+        asyncio.create_task(_run_loop(WEEKLY_CRON, "weekly_metadata", jobs.run_metadata_weekly)),
+    ]
+    logger.info("Internal cron started (daily=%s, weekly=%s)", DAILY_CRON, WEEKLY_CRON)
+    return tasks

app/jobs.py

@@ -0,0 +1,76 @@
+"""
+Background jobs for HuggingFace Spaces (triggered via secured cron HTTP endpoints).
+"""
+
+import asyncio
+import logging
+import os
+import sys
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Scripts live beside app/ in the Docker image
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+_job_lock = asyncio.Lock()
+
+
+async def run_daily_forecasts() -> dict:
+    """Generate and cache forecasts for tracked stocks (all countries)."""
+    from scripts.daily_pipeline import run_daily_pipeline
+
+    await run_daily_pipeline()
+    return {"job": "daily_forecasts", "status": "completed"}
+
+
+async def run_metadata_daily() -> dict:
+    """Trading status + listing checks."""
+    from app.scheduler import scheduler
+
+    await scheduler.initialize()
+    try:
+        await scheduler.daily_sync()
+        return {"job": "metadata_daily", "status": "completed"}
+    finally:
+        await scheduler.shutdown()
+
+
+async def run_metadata_weekly() -> dict:
+    """Full exchange directory refresh."""
+    from app.scheduler import scheduler
+
+    await scheduler.initialize()
+    try:
+        await scheduler.weekly_full_sync()
+        return {"job": "metadata_weekly", "status": "completed"}
+    finally:
+        await scheduler.shutdown()
+
+
+async def run_full_daily_job() -> dict:
+    """
+    HuggingFace daily cron (recommended): forecast pipeline only.
+    Metadata refresh runs on weekly cron to avoid Yahoo 429 rate limits.
+    Set CRON_INCLUDE_METADATA_DAILY=true to also run metadata daily.
+    """
+    async with _job_lock:
+        result: dict = {"status": "completed"}
+        if os.getenv("CRON_INCLUDE_METADATA_DAILY", "").lower() in ("1", "true", "yes"):
+            result["metadata"] = await run_metadata_daily()
+        result["forecasts"] = await run_daily_forecasts()
+        return result
+
+
+async def run_job_in_background(coro_factory):
+    """Fire-and-forget for long jobs (HF cron HTTP must return quickly)."""
+    task = asyncio.create_task(coro_factory())
+
+    def _done(t: asyncio.Task):
+        try:
+            t.result()
+            logger.info("Background job finished OK")
+        except Exception as e:
+            logger.error(f"Background job failed: {e}")
+
+    task.add_done_callback(_done)

app/main.py

@@ -7,6 +7,7 @@ from fastapi import FastAPI, HTTPException, Query
 from fastapi.middleware.cors import CORSMiddleware
 from contextlib import asynccontextmanager
 import os
+from typing import Optional
 from dotenv import load_dotenv
 
 from app.services.timesfm_service import TimesFMService
@@ -14,12 +15,15 @@ from app.services.data_service import StockDataService
 from app.services.chart_service import ChartService
 from app.services.database_service import DatabaseService
 from app.models.schemas import ForecastResponse, StockInfo, HealthResponse
+from app.cron import router as cron_router
+from app.internal_cron import start_internal_cron
 
 # Load environment variables
 load_dotenv()
 
 # Global service instances
 timesfm_service = None
+_cron_tasks: list = []
 data_service = None
 chart_service = None
 db_service = None
@@ -43,11 +47,14 @@ async def lifespan(app: FastAPI):
     # Initialize TimesFM model (this takes time)
     print("🤖 Loading TimesFM 2.5 model...")
     timesfm_service = TimesFMService(
-        model_id="google/timesfm-2.5-200m-pytorch",
-        device="cpu"  # Use CPU for free tier compatibility
+        model_id=os.getenv("TIMESFM_MODEL", "google/timesfm-2.5-200m-transformers"),
+        device=os.getenv("DEVICE", "cpu"),
     )
     await timesfm_service.load_model()
-    print("✅ TimesFM model loaded")
+    if timesfm_service.model is not None:
+        print("✅ TimesFM model loaded")
+    else:
+        print("⚠️ TimesFM unavailable — using statistical fallback forecasts")
     
     # Initialize data and chart services
     data_service = StockDataService()
@@ -55,10 +62,15 @@ async def lifespan(app: FastAPI):
     print("✅ Data and chart services ready")
     
     print("🎉 All services initialized successfully")
+
+    global _cron_tasks
+    _cron_tasks = await start_internal_cron()
     
     yield
     
     # Cleanup
+    for t in _cron_tasks:
+        t.cancel()
     print("👋 Shutting down services...")
     await db_service.close()
 
@@ -80,6 +92,8 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+app.include_router(cron_router)
+
 
 @app.get("/health", response_model=HealthResponse)
 async def health_check():
@@ -112,41 +126,48 @@ async def get_forecast(
         if cached_forecast:
             return cached_forecast
         
-        # Fetch historical data
-        stock_data = await data_service.get_stock_data(symbol, period="2y")
-        if not stock_data or len(stock_data) < 100:
+        # Fetch historical data (yfinance DataFrame)
+        stock_df = await data_service.get_stock_data(symbol, period="2y")
+        if stock_df is None or len(stock_df) < 100:
             raise HTTPException(
                 status_code=404,
                 detail=f"Insufficient historical data for {symbol}"
             )
-        
+
+        close_col = "Close" if "Close" in stock_df.columns else "close"
+        close_series = stock_df[close_col]
+        current_price = float(close_series.iloc[-1])
+        stock_name = stock_df.attrs.get("name", symbol)
+        stock_exchange = stock_df.attrs.get("exchange", "UNKNOWN")
+        stock_currency = stock_df.attrs.get("currency", "USD")
+        last_updated = stock_df.index[-1].isoformat()
+
         # Generate forecast
         forecast_result = await timesfm_service.predict(
-            historical_prices=stock_data['close'].tolist(),
+            historical_prices=close_series.tolist(),
             horizon=horizon
         )
         
         # Generate chart
         chart_svg = chart_service.generate_forecast_chart(
             symbol=symbol,
-            historical_prices=stock_data['close'].tolist()[-60:],  # Last 60 days
+            historical_prices=close_series.tolist()[-60:],  # Last 60 days
             forecast=forecast_result,
-            current_price=stock_data['close'].iloc[-1]
+            current_price=current_price
         )
         
         # Prepare response
         response = ForecastResponse(
             symbol=symbol,
-            name=stock_data.get('name', symbol),
-            exchange=stock_data.get('exchange', 'UNKNOWN'),
-            currency=stock_data.get('currency', 'USD'),
-            current_price=float(stock_data['close'].iloc[-1]),
-            last_updated=stock_data['date'].iloc[-1].isoformat(),
+            name=stock_name,
+            exchange=stock_exchange,
+            currency=stock_currency,
+            current_price=current_price,
+            last_updated=last_updated,
             horizon_days=horizon,
             point_forecast=float(forecast_result['point_forecast'][-1]),
             percentage_change=float(
-                ((forecast_result['point_forecast'][-1] - stock_data['close'].iloc[-1]) 
-                 / stock_data['close'].iloc[-1]) * 100
+                ((forecast_result['point_forecast'][-1] - current_price) / current_price) * 100
             ),
             quantiles={
                 'p10': [float(x) for x in forecast_result['quantiles']['p10']],

app/scheduler.py

@@ -76,7 +76,8 @@ class DataSyncScheduler:
         try:
             # 1. Check trading status for sample of stocks
             logger.info("\n📊 Checking trading status for sample stocks...")
-            await self.check_trading_status_sample(limit=100)
+            status_limit = int(os.getenv("METADATA_STATUS_CHECK_LIMIT", "15"))
+            await self.check_trading_status_sample(limit=status_limit)
             
             # 2. Check for new listings (incremental)
             logger.info("\n🔍 Checking for new listings...")
@@ -201,22 +202,16 @@ class DataSyncScheduler:
                 try:
                     yf_symbol = f"{stock['symbol']}{stock['yfinance_suffix']}"
                     
-                    # Try to get stock info first (lighter than historical data)
-                    info = await self.data_service.get_stock_info(yf_symbol)
-                    
-                    if info is None or not info.get('current_price'):
+                    # Use chart API (same path as forecasts) — avoids yfinance .info 429 storms
+                    hist = await self.data_service.get_stock_data(yf_symbol, period="5d")
+                    if hist is None or len(hist) < 1:
                         await self.db.update_trading_status(stock['symbol'], 'inactive')
                         inactive_count += 1
-                        logger.debug(f"   ⚠️  {stock['symbol']} - inactive (no price data)")
                     else:
-                        # Stock has price data, mark as active
                         await self.db.update_trading_status(stock['symbol'], 'active')
                         updated_count += 1
-                        logger.debug(f"   ✅ {stock['symbol']} - active (price: {info['current_price']})")
-                    
-                    # Rate limiting
-                    if checked_count % 5 == 0:
-                        await asyncio.sleep(0.3)
+
+                    await asyncio.sleep(float(os.getenv("METADATA_STOCK_DELAY", "1.0")))
                         
                 except Exception as e:
                     # Don't fail the entire sync if one stock fails
@@ -347,10 +342,6 @@ class DataSyncScheduler:
         self.is_running = False
 
 
-# Global scheduler instance
-scheduler = DataSyncScheduler()
-
-
 async def run_scheduler_loop():
     """Main scheduler loop - runs continuously"""
     await scheduler.initialize()

app/services/data_service.py

@@ -41,6 +41,18 @@ class StockDataService:
         self.cache = {}  # In-memory cache (faster)
         self._session = None
         self._setup_cache_dir()
+        self._init_yfinance_session()
+
+    def _init_yfinance_session(self):
+        """Use curl_cffi browser impersonation when available (bypasses Yahoo blocks)."""
+        try:
+            from curl_cffi import requests as curl_requests
+
+            self._session = curl_requests.Session(impersonate="chrome120")
+            logger.info("yfinance session: curl_cffi chrome120 impersonation enabled")
+        except ImportError:
+            logger.warning("curl_cffi not available — yfinance may be rate-limited")
+            self._session = None
 
     def _setup_cache_dir(self):
         """Create cache directory if it doesn't exist"""
@@ -64,23 +76,36 @@ class StockDataService:
     def _read_from_disk_cache(self, symbol: str, period: str) -> Optional[pd.DataFrame]:
         """Read data from disk cache"""
         cache_path = self._get_cache_path(symbol, period)
-        if self._is_cache_fresh(cache_path):
+        json_path = cache_path.with_suffix(".json")
+        for path in (cache_path, json_path):
+            if not self._is_cache_fresh(path):
+                continue
             try:
-                df = pd.read_parquet(cache_path)
+                if path.suffix == ".json":
+                    df = pd.read_json(path)
+                    if "Date" in df.columns:
+                        df = df.set_index("Date")
+                    df.index = pd.to_datetime(df.index)
+                else:
+                    df = pd.read_parquet(path)
                 logger.debug(f"Disk cache hit for {symbol} ({period})")
                 return df
             except Exception as e:
-                logger.warning(f"Failed to read disk cache for {symbol}: {e}")
+                logger.debug(f"Disk cache read failed for {symbol}: {e}")
         return None
 
     def _write_to_disk_cache(self, symbol: str, period: str, df: pd.DataFrame):
-        """Write data to disk cache"""
+        """Write data to disk cache (Parquet if pyarrow available, else JSON)."""
         try:
             cache_path = self._get_cache_path(symbol, period)
-            df.to_parquet(cache_path)
+            try:
+                df.to_parquet(cache_path)
+            except ImportError:
+                json_path = cache_path.with_suffix(".json")
+                df.reset_index().to_json(json_path, orient="records", date_format="iso")
             logger.debug(f"Cached {symbol} ({period}) to disk")
         except Exception as e:
-            logger.warning(f"Failed to write disk cache for {symbol}: {e}")
+            logger.debug(f"Skipped disk cache for {symbol}: {e}")
 
     def _clear_expired_cache(self):
         """Clear expired cache entries"""
@@ -97,6 +122,64 @@ class StockDataService:
         except Exception as e:
             logger.warning(f"Cache cleanup failed: {e}")
 
+    @staticmethod
+    def _period_to_yahoo_range(period: str) -> str:
+        mapping = {
+            "1mo": "1mo",
+            "3mo": "3mo",
+            "6mo": "6mo",
+            "1y": "1y",
+            "2y": "2y",
+            "5y": "5y",
+            "10y": "10y",
+            "max": "max",
+        }
+        return mapping.get(period, "2y")
+
+    def _yahoo_chart_fetch(self, symbol: str, period: str = "2y") -> Optional[pd.DataFrame]:
+        """Fallback: Yahoo v8 chart API via curl_cffi (works when yfinance library is blocked)."""
+        if not self._session:
+            return None
+
+        try:
+            range_param = self._period_to_yahoo_range(period)
+            url = (
+                "https://query1.finance.yahoo.com/v8/finance/chart/"
+                f"{symbol}?interval=1d&range={range_param}"
+            )
+            response = self._session.get(url, timeout=30)
+            response.raise_for_status()
+            payload = response.json()
+
+            result = payload.get("chart", {}).get("result", [])
+            if not result:
+                return None
+
+            chart = result[0]
+            timestamps = chart.get("timestamp") or []
+            quotes = chart.get("indicators", {}).get("quote", [{}])[0]
+            closes = quotes.get("close") or []
+
+            rows = [
+                (pd.Timestamp(ts, unit="s"), close)
+                for ts, close in zip(timestamps, closes)
+                if close is not None
+            ]
+            if len(rows) < 30:
+                return None
+
+            df = pd.DataFrame(rows, columns=["Date", "Close"]).set_index("Date")
+            meta = chart.get("meta", {})
+            df.attrs["name"] = meta.get("longName") or meta.get("shortName") or symbol
+            df.attrs["exchange"] = meta.get("exchangeName", "UNKNOWN")
+            df.attrs["currency"] = meta.get("currency", "USD")
+            logger.info(f"Yahoo chart API fallback succeeded for {symbol} ({len(df)} rows)")
+            return df
+
+        except Exception as e:
+            logger.warning(f"Yahoo chart API fallback failed for {symbol}: {e}")
+            return None
+
     def _yfinance_fetch(
         self, symbol: str, period: str = "2y"
     ) -> Optional[pd.DataFrame]:
@@ -104,12 +187,12 @@ class StockDataService:
         This runs inside ThreadPoolExecutor to enable parallelism.
         """
         try:
-            ticker = yf.Ticker(symbol)
+            ticker = yf.Ticker(symbol, session=self._session) if self._session else yf.Ticker(symbol)
             df = ticker.history(period=period)
 
             if df.empty:
-                logger.warning(f"No data found for {symbol}")
-                return None
+                logger.warning(f"yfinance empty for {symbol}, trying Yahoo chart API")
+                return self._yahoo_chart_fetch(symbol, period)
 
             # Ensure 'Close' column exists
             if 'Close' not in df.columns:
@@ -131,7 +214,7 @@ class StockDataService:
 
         except Exception as e:
             logger.error(f"yfinance fetch failed for {symbol}: {e}")
-            return None
+            return self._yahoo_chart_fetch(symbol, period)
 
     @retry(
         stop=stop_after_attempt(4),
@@ -202,7 +285,7 @@ class StockDataService:
             loop = asyncio.get_event_loop()
 
             def _fetch_info():
-                ticker = yf.Ticker(symbol)
+                ticker = yf.Ticker(symbol, session=self._session) if self._session else yf.Ticker(symbol)
                 info = ticker.info
                 if not info or 'symbol' not in info:
                     return None

app/services/timesfm_service.py

@@ -29,9 +29,12 @@ class TimesFMService:
         self.available = TIMESFM_AVAILABLE
         
     async def load_model(self):
-        """Load the TimesFM model from HuggingFace"""
+        """Load the TimesFM model from HuggingFace (or use statistical fallback)"""
         if not self.available:
-            raise RuntimeError("TimesFM 2.5 not available. Install latest transformers or check import")
+            logger.warning(
+                "TimesFM 2.5 not available in this environment — API will use statistical fallback forecasts"
+            )
+            return
 
         try:
             logger.info(f"Loading TimesFM model: {self.model_id}")
@@ -144,8 +147,8 @@ class TimesFMService:
             point_forecast = []
             current_price = last_price
 
-            for _ in range(horizon):
-                current_price = current_price * (1 + daily_drift + np.random.normal(0, volatility * 0.5))
+            for day in range(horizon):
+                current_price = current_price * (1 + daily_drift)
                 point_forecast.append(current_price)
 
             # Generate confidence intervals

scripts/backfill_sectors.py

@@ -0,0 +1,82 @@
+"""
+Backfill missing sector on stocks via yfinance (batch, rate-limited).
+Run: python scripts/backfill_sectors.py [--limit 500]
+"""
+
+import argparse
+import asyncio
+import os
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from app.services.database_service import DatabaseService
+
+try:
+    import yfinance as yf
+except ImportError:
+    print("pip install yfinance")
+    sys.exit(1)
+
+
+def yf_ticker(symbol: str, suffix: str) -> str:
+    s = symbol.upper()
+    if not suffix:
+        return s
+    suf = suffix if suffix.startswith(".") else f".{suffix}"
+    return s if s.endswith(suf) else f"{s}{suf}"
+
+
+async def main(limit: int) -> None:
+    turso_url = os.getenv("TURSO_URL")
+    turso_token = os.getenv("TURSO_TOKEN")
+    if not turso_url or not turso_token:
+        print("Set TURSO_URL and TURSO_TOKEN in backend/.env")
+        sys.exit(1)
+
+    db = DatabaseService(url=turso_url, token=turso_token)
+    await db.initialize()
+
+    rows = await db._execute(
+        """
+        SELECT symbol, country, yfinance_suffix, name
+        FROM stocks
+        WHERE sector IS NULL OR sector = ''
+        ORDER BY market_cap DESC NULLS LAST
+        LIMIT ?
+        """,
+        [limit],
+    )
+
+    print(f"Backfilling sector for {len(rows)} stocks...")
+    updated = 0
+    for i, row in enumerate(rows, 1):
+        sym = yf_ticker(row["symbol"], row.get("yfinance_suffix") or "")
+        try:
+            info = yf.Ticker(sym).info or {}
+            sector = info.get("sector") or info.get("industry")
+            if sector:
+                await db._execute(
+                    "UPDATE stocks SET sector = ? WHERE symbol = ? AND country = ?",
+                    [str(sector)[:120], row["symbol"], row["country"]],
+                )
+                updated += 1
+                print(f"  [{i}/{len(rows)}] {sym} -> {sector}")
+        except Exception as e:
+            print(f"  [{i}/{len(rows)}] {sym} skip: {e}")
+        time.sleep(0.25)
+
+    print(f"Done. Updated {updated} rows.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--limit", type=int, default=500)
+    args = parser.parse_args()
+    asyncio.run(main(args.limit))

scripts/configure_hf_space.py

@@ -0,0 +1,70 @@
+"""Configure HuggingFace Space secrets/variables for free-tier deploy (no HF Jobs)."""
+
+from __future__ import annotations
+
+import os
+import secrets
+import sys
+from pathlib import Path
+
+from dotenv import dotenv_values
+from huggingface_hub import HfApi
+
+REPO_ID = "cramamoorthy/stock-prediction-api"
+
+
+def main() -> int:
+    env_path = Path(__file__).resolve().parent.parent / ".env"
+    env = dotenv_values(env_path)
+    api = HfApi()
+
+    turso_url = env.get("TURSO_URL") or os.getenv("TURSO_URL")
+    turso_token = env.get("TURSO_TOKEN") or os.getenv("TURSO_TOKEN")
+    if not turso_url or not turso_token:
+        print("Missing TURSO_URL / TURSO_TOKEN in backend/.env", file=sys.stderr)
+        return 1
+
+    cron_secret = secrets.token_urlsafe(32)
+
+    print("Setting Space secrets and variables...")
+    api.add_space_secret(REPO_ID, "TURSO_URL", turso_url)
+    api.add_space_secret(REPO_ID, "TURSO_TOKEN", turso_token)
+    api.add_space_secret(REPO_ID, "CRON_SECRET", cron_secret)
+    api.add_space_variable(REPO_ID, "DAILY_FORECAST_LIMIT_PER_COUNTRY", "50")
+
+    api.add_space_variable(REPO_ID, "ENABLE_INTERNAL_CRON", "true")
+
+    import requests
+    from huggingface_hub.utils import build_hf_headers
+    from huggingface_hub import get_token
+
+    r = requests.get(
+        f"https://huggingface.co/api/spaces/{REPO_ID}/secrets",
+        headers=build_hf_headers(token=get_token()),
+    )
+    secret_keys = list(r.json().keys()) if r.ok else ["TURSO_URL", "TURSO_TOKEN", "CRON_SECRET"]
+    print("Space secrets:", ", ".join(secret_keys))
+    print("Variables: DAILY_FORECAST_LIMIT_PER_COUNTRY=50, ENABLE_INTERNAL_CRON=true")
+    print("Internal cron: daily 0 21 * * 1-5 UTC, weekly 0 2 * * 0 UTC (while Space is running)")
+    print("Optional UI: Settings → Scheduler — same paths/headers to wake sleeping Space.")
+    print("Space will restart after secret changes.")
+
+    # Persist cron secret locally for manual triggers (gitignored via .env)
+    lines = env_path.read_text(encoding="utf-8").splitlines()
+    out: list[str] = []
+    replaced = False
+    for line in lines:
+        if line.startswith("CRON_SECRET="):
+            out.append(f"CRON_SECRET={cron_secret}")
+            replaced = True
+        else:
+            out.append(line)
+    if not replaced:
+        out.append(f"CRON_SECRET={cron_secret}")
+    env_path.write_text("\n".join(out) + "\n", encoding="utf-8")
+    print("Updated backend/.env CRON_SECRET for local curl tests.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/daily_pipeline.py

@@ -44,7 +44,7 @@ HORIZONS = [1, 5, 20, 60]
 
 # Max stocks per country per run (limit to manage pipeline duration)
 # With 7,000+ stocks, stagger across multiple runs or increase for overnight processing
-MAX_STOCKS_PER_COUNTRY = 10000  # Set high enough to cover US all stocks (~6,076)
+MAX_STOCKS_PER_COUNTRY = int(os.getenv("DAILY_FORECAST_LIMIT_PER_COUNTRY", "10000"))
 
 # Delay between individual stock fetches (seconds)
 STOCK_DELAY = 0.3
@@ -162,7 +162,7 @@ async def process_stock(
             }
 
             # Cache in database
-            await db.cache_forecast(clean_symbol, horizon, forecast_data)
+            await db.cache_forecast(yf_symbol.upper(), horizon, forecast_data)
 
         print(f"[OK] {len(HORIZONS)} forecasts cached")
         return True

scripts/seed_sp500.py

@@ -0,0 +1,12 @@
+"""
+S&P / US starter seed — delegates to verified stock seeder (US subset).
+Documented in QUICKSTART.md as the fast path for local development.
+"""
+
+import asyncio
+import runpy
+from pathlib import Path
+
+if __name__ == "__main__":
+    script = Path(__file__).parent / "seed_verified_stocks.py"
+    runpy.run_path(str(script), run_name="__main__")

scripts/setup_free_wake_cron.py

@@ -0,0 +1,263 @@
+"""Configure free wake cron: HF Space Scheduler (Playwright) or cron-job.org API."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+import requests
+from dotenv import dotenv_values
+from huggingface_hub import get_token
+
+SPACE = "cramamoorthy/stock-prediction-api"
+SPACE_URL = "https://cramamoorthy-stock-prediction-api.hf.space"
+SETTINGS = f"https://huggingface.co/spaces/{SPACE}/settings"
+CRON_API = "https://api.cron-job.org"
+
+
+def _secret() -> str:
+    v = dotenv_values(Path(__file__).resolve().parent.parent / ".env").get("CRON_SECRET")
+    if not v:
+        raise SystemExit("CRON_SECRET missing in backend/.env — run configure_hf_space.py first")
+    return v
+
+
+def _job_schedule_daily() -> dict:
+    return {
+        "timezone": "UTC",
+        "expiresAt": 0,
+        "hours": [21],
+        "minutes": [0],
+        "mdays": [-1],
+        "months": [-1],
+        "wdays": [1, 2, 3, 4, 5],
+    }
+
+
+def _job_schedule_weekly() -> dict:
+    return {
+        "timezone": "UTC",
+        "expiresAt": 0,
+        "hours": [2],
+        "minutes": [0],
+        "mdays": [-1],
+        "months": [-1],
+        "wdays": [0],
+    }
+
+
+def setup_cron_job_org(secret: str) -> bool:
+    api_key = os.getenv("CRON_JOB_ORG_API_KEY") or dotenv_values(
+        Path(__file__).resolve().parent.parent / ".env"
+    ).get("CRON_JOB_ORG_API_KEY")
+    if not api_key:
+        return False
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    existing = requests.get(f"{CRON_API}/jobs", headers=headers, timeout=30)
+    existing.raise_for_status()
+    titles = {j.get("title") for j in existing.json().get("jobs", [])}
+
+    jobs = [
+        (
+            "stock-prediction-daily",
+            f"{SPACE_URL}/api/v1/cron/daily",
+            _job_schedule_daily(),
+        ),
+        (
+            "stock-prediction-weekly",
+            f"{SPACE_URL}/api/v1/cron/metadata/weekly",
+            _job_schedule_weekly(),
+        ),
+    ]
+
+    for title, url, schedule in jobs:
+        if title in titles:
+            print(f"cron-job.org: {title} already exists")
+            continue
+        payload = {
+            "job": {
+                "title": title,
+                "url": url,
+                "enabled": True,
+                "requestMethod": 1,
+                "schedule": schedule,
+                "extendedData": {
+                    "headers": {"X-Cron-Secret": secret},
+                },
+            }
+        }
+        r = requests.put(f"{CRON_API}/jobs", headers=headers, json=payload, timeout=30)
+        r.raise_for_status()
+        print(f"cron-job.org: created {title} (id={r.json().get('jobId')})")
+    return True
+
+
+def setup_hf_scheduler_playwright(secret: str) -> bool:
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        return False
+
+    token = get_token()
+    if not token:
+        return False
+
+    captured: list[dict] = []
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(
+            user_agent=(
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+            )
+        )
+
+        def on_request(req):
+            u = req.url
+            if "huggingface.co" in u and req.method in ("POST", "PUT", "PATCH", "DELETE"):
+                if any(x in u.lower() for x in ("schedul", "cron", "webhook", "request")):
+                    captured.append({"method": req.method, "url": u, "post": req.post_data})
+
+        page = context.new_page()
+        page.on("request", on_request)
+
+        # Token login flow used by HF hub
+        page.goto("https://huggingface.co/login", wait_until="domcontentloaded", timeout=60_000)
+        page.goto(
+            f"https://huggingface.co/login?token={token}&next=/spaces/{SPACE}/settings",
+            wait_until="domcontentloaded",
+            timeout=120_000,
+        )
+        page.wait_for_timeout(3000)
+        if "login" in page.url.lower() and "settings" not in page.url:
+            page.goto(SETTINGS, wait_until="domcontentloaded", timeout=120_000)
+            page.wait_for_timeout(2000)
+
+        if "403" in (page.title() or "") or "ERROR" in (page.inner_text("body")[:200] if page.locator("body").count() else ""):
+            browser.close()
+            return False
+
+        # Open Scheduler tab
+        for sel in [
+            "a[href*='scheduler']",
+            "button:has-text('Scheduler')",
+            "text=Scheduler",
+        ]:
+            loc = page.locator(sel)
+            if loc.count():
+                loc.first.click()
+                page.wait_for_timeout(2000)
+                break
+
+        def add_job(title: str, cron_expr: str, path: str) -> None:
+            add_btn = page.get_by_role("button", name=lambda n: n and any(
+                w in (n or "").lower() for w in ("add", "new", "create")
+            ))
+            if add_btn.count():
+                add_btn.first.click()
+                page.wait_for_timeout(1000)
+
+            page.locator("input").filter(has_text="").all()  # noop wake
+            for inp in page.locator("input:visible").all():
+                ph = (inp.get_attribute("placeholder") or "").lower()
+                name = (inp.get_attribute("name") or "").lower()
+                if "cron" in ph or "schedule" in ph or "cron" in name:
+                    inp.fill(cron_expr)
+                elif "path" in ph or "path" in name or "url" in ph:
+                    inp.fill(path)
+
+            page.locator("select:visible").first.select_option("POST") if page.locator("select:visible").count() else None
+
+            # headers
+            page.get_by_text("Header", exact=False).first.click() if page.get_by_text("Header").count() else None
+            header_inputs = page.locator("input:visible")
+            for i, inp in enumerate(header_inputs.all()):
+                val = inp.input_value()
+                if not val and i == 0:
+                    inp.fill("X-Cron-Secret")
+                elif val == "X-Cron-Secret" or (not val and i == 1):
+                    inp.fill(secret)
+
+            save = page.get_by_role("button", name=lambda n: n and any(
+                w in (n or "").lower() for w in ("save", "create", "add", "submit")
+            ))
+            if save.count():
+                save.first.click()
+                page.wait_for_timeout(3000)
+            print(f"HF Scheduler UI: attempted {title}")
+
+        add_job("daily", "0 21 * * 1-5", "/api/v1/cron/daily")
+        add_job("weekly", "0 2 * * 0", "/api/v1/cron/metadata/weekly")
+
+        out = Path(__file__).parent / "hf_scheduler_capture.json"
+        out.write_text(json.dumps(captured, indent=2), encoding="utf-8")
+
+        # Verify jobs listed in UI
+        body = page.inner_text("body") if page.locator("body").count() else ""
+        ok = "/api/v1/cron/daily" in body or "cron/daily" in body
+        browser.close()
+        return ok
+
+    return False
+
+
+def setup_hf_scheduler_api(secret: str) -> bool:
+    """Try discovered/internal HF endpoints."""
+    from huggingface_hub.utils import build_hf_headers
+
+    token = get_token()
+    if not token:
+        return False
+    h = build_hf_headers(token=token)
+    h["Content-Type"] = "application/json"
+    base = f"https://huggingface.co/api/spaces/{SPACE}"
+
+    payloads = [
+        {
+            "schedule": "0 21 * * 1-5",
+            "method": "POST",
+            "path": "/api/v1/cron/daily",
+            "headers": [{"name": "X-Cron-Secret", "value": secret}],
+        },
+        {
+            "schedule": "0 2 * * 0",
+            "method": "POST",
+            "path": "/api/v1/cron/metadata/weekly",
+            "headers": [{"name": "X-Cron-Secret", "value": secret}],
+        },
+    ]
+
+    for path_suffix in ("scheduled-requests", "scheduler/requests", "schedule"):
+        for p in payloads:
+            r = requests.post(f"{base}/{path_suffix}", headers=h, json=p, timeout=30)
+            if r.status_code in (200, 201):
+                print(f"HF API {path_suffix}: OK")
+                return True
+    return False
+
+
+def main() -> int:
+    import subprocess
+    root = Path(__file__).resolve().parent.parent.parent
+    script = root / "scripts" / "deploy-hf-cron-worker.ps1"
+    if script.exists():
+        subprocess.run(
+            ["powershell", "-NoProfile", "-File", str(script)],
+            cwd=str(root),
+            check=True,
+        )
+        print("Done: Cloudflare Worker cron (free).")
+        return 0
+    print(f"Missing {script}", file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

start-api.ps1

@@ -0,0 +1,18 @@
+# Start FastAPI locally (single instance on port 8000)
+$ErrorActionPreference = "Stop"
+Set-Location $PSScriptRoot
+
+$existing = Get-NetTCPConnection -LocalPort 8000 -State Listen -ErrorAction SilentlyContinue
+if ($existing) {
+    $pid = $existing[0].OwningProcess
+    $proc = Get-Process -Id $pid -ErrorAction SilentlyContinue
+    if ($proc -and $proc.Path -like "*python*") {
+        Write-Host "Port 8000 already in use by API (PID $pid). Not restarting."
+        exit 0
+    }
+    Stop-Process -Id $pid -Force -ErrorAction SilentlyContinue
+    Start-Sleep -Seconds 1
+}
+
+Write-Host "Starting Stock Prediction API on http://127.0.0.1:8000"
+& .\venv\Scripts\python.exe -m uvicorn app.main:app --reload --host 127.0.0.1 --port 8000

test_yfinance_quick.py

@@ -0,0 +1,9 @@
+import asyncio
+from app.services.data_service import StockDataService
+
+async def main():
+    s = StockDataService()
+    df = await s.get_stock_data("AAPL", "2y", force_refresh=True)
+    print("result", None if df is None else len(df), list(df.columns) if df is not None else None)
+
+asyncio.run(main())