scripts/daily_hf_sync.sh

#!/bin/bash
# Daily Mac -> Hugging Face synchronization.
#
# This refreshes local precomputed predictions and pushes publishable results to
# the HF Dataset queue used by the Hugging Face Space.
set -euo pipefail

REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
LOG_DIR="$REPO_DIR/logs"
LOG_FILE="${DAILY_HF_SYNC_LOG:-$LOG_DIR/daily_hf_sync.log}"
LOCK_DIR="$LOG_DIR/daily_hf_sync.lock"
PYTHON="$REPO_DIR/venv/bin/python"

mkdir -p "$LOG_DIR"
cd "$REPO_DIR"

if ! mkdir "$LOCK_DIR" 2>/dev/null; then
  echo "==== $(date '+%Y-%m-%d %H:%M:%S %Z') daily HF sync skipped: another run is active ====" | tee -a "$LOG_FILE"
  exit 0
fi
trap 'rmdir "$LOCK_DIR"' EXIT

if [ ! -x "$PYTHON" ]; then
  echo "ERROR: Python not found at $PYTHON" | tee -a "$LOG_FILE"
  exit 1
fi

{
  echo "==== $(date '+%Y-%m-%d %H:%M:%S %Z') daily HF sync start ===="
  echo "Repo: $REPO_DIR"

  export PYTHONUNBUFFERED="${PYTHONUNBUFFERED:-1}"
  export ENABLE_MULTI_FACTOR_WEIGHT_OVERLAY="${ENABLE_MULTI_FACTOR_WEIGHT_OVERLAY:-1}"
  export ENABLE_GEMINI_NEWS="${ENABLE_GEMINI_NEWS:-0}"
  export ENABLE_PUBLIC_NEWS_FALLBACK="${ENABLE_PUBLIC_NEWS_FALLBACK:-1}"

  HOT_LIMIT="${HOT_SYNC_LIMIT:-50}"
  QUERY_DAYS="${QUERY_SYNC_DAYS:-30}"
  QUERY_LIMIT="${QUERY_SYNC_LIMIT:-50}"
  HORIZON="${PREDICTION_HORIZON:-5}"
  MONTHS="${BACKTEST_MONTHS:-6}"
  INCLUDE_INSTITUTIONAL="${INCLUDE_INSTITUTIONAL_SYNC:-1}"
  NEWS_FLAG=""
  INSTITUTIONAL_FLAG=""
  if [ "${INCLUDE_NEWS_SYNC:-1}" != "1" ]; then
    NEWS_FLAG="--no-news"
  fi
  if [ "$INCLUDE_INSTITUTIONAL" = "1" ]; then
    INSTITUTIONAL_FLAG="--include-institutional"
  fi

  echo "Config: HOT_SYNC_LIMIT=$HOT_LIMIT QUERY_SYNC_DAYS=$QUERY_DAYS QUERY_SYNC_LIMIT=$QUERY_LIMIT PREDICTION_HORIZON=$HORIZON INCLUDE_NEWS_SYNC=${INCLUDE_NEWS_SYNC:-1} INCLUDE_INSTITUTIONAL_SYNC=$INCLUDE_INSTITUTIONAL"

  if [ "${DRY_RUN:-0}" = "1" ]; then
    echo "DRY_RUN=1, commands were not executed."
    echo "$PYTHON scripts/precompute_hot20.py --limit $HOT_LIMIT --horizon $HORIZON --months $MONTHS --prediction-only --push $NEWS_FLAG $INSTITUTIONAL_FLAG"
    echo "$PYTHON scripts/precompute_queried_stocks.py --days $QUERY_DAYS --limit $QUERY_LIMIT --horizon $HORIZON --months $MONTHS --push $NEWS_FLAG $INSTITUTIONAL_FLAG"
    echo "$PYTHON scripts/hf_queue_worker.py"
    exit 0
  fi

  "$PYTHON" scripts/precompute_hot20.py \
    --limit "$HOT_LIMIT" \
    --horizon "$HORIZON" \
    --months "$MONTHS" \
    --prediction-only \
    --push \
    $NEWS_FLAG \
    $INSTITUTIONAL_FLAG \
    --output-json docs/hot20_precompute_backtest.json \
    --output-md docs/hot20_precompute_backtest.md \
    --local-cache data/precomputed_predictions.json

  "$PYTHON" scripts/precompute_queried_stocks.py \
    --days "$QUERY_DAYS" \
    --limit "$QUERY_LIMIT" \
    --horizon "$HORIZON" \
    --months "$MONTHS" \
    --push \
    $NEWS_FLAG \
    $INSTITUTIONAL_FLAG \
    --output-json docs/queried_precompute.json \
    --output-md docs/queried_precompute.md \
    --local-cache data/precomputed_predictions.json

  "$PYTHON" scripts/hf_queue_worker.py
  "$PYTHON" scripts/hf_queue_worker.py status || true

  echo "==== $(date '+%Y-%m-%d %H:%M:%S %Z') daily HF sync done ===="
} 2>&1 | tee -a "$LOG_FILE"