scripts/run_10h_loop.sh

#!/usr/bin/env bash
# 10-hour rolling loop: re-scan both data pools repeatedly so select_template
# naturally rotates through under-used chart_name variations (variation.json
# weights each pick by usage frequency, so the same input may end up rendered
# with different variations across rounds).
#
# Stop policy (graceful): before each round, check elapsed wall clock; if >=
# DEADLINE_SEC, do NOT start a new round. The currently running round is
# always allowed to finish.
#
# Output: APPENDS into output/line_pie_donut_chart_only/<pool>/ (same dir as
# the foundational run). Filenames carry an int timestamp so they never
# collide.
#
# Logs:
#   logs/10h_run_round_<N>_<pool>.log    per-round per-pool pipeline output
#   logs/10h_run_summary.log             this wrapper's own progress

set -u
set -o pipefail

cd "$(dirname "$0")/.."
ROOT="$(pwd)"

SUMMARY=logs/10h_run_summary.log
mkdir -p logs output/line_pie_donut_chart_only

log() { printf '[%s] %s\n' "$(date -Iseconds)" "$*" | tee -a "$SUMMARY"; }

log "==== 10h loop launcher ===="
log "cwd=$ROOT"

# -------- 1. wait for current line_pie_donut session to finish --------
PRIOR_SESSION="${PRIOR_SESSION:-line_pie_donut}"
if tmux has-session -t "$PRIOR_SESSION" 2>/dev/null; then
  log "waiting for prior tmux session '$PRIOR_SESSION' to finish..."
  while tmux has-session -t "$PRIOR_SESSION" 2>/dev/null; do
    sleep 60
  done
  log "prior session '$PRIOR_SESSION' is gone, continuing"
else
  log "prior session '$PRIOR_SESSION' not running, starting immediately"
fi

# -------- 2. config --------
if [[ ! -s allowed_chart_types.json ]]; then
  log "ERROR: allowed_chart_types.json missing or empty; aborting"
  exit 1
fi
log "whitelist (will be re-read by every worker via mtime cache):"
cat allowed_chart_types.json | tee -a "$SUMMARY"

export PUPPETEER_EXECUTABLE_PATH="${PUPPETEER_EXECUTABLE_PATH:-/usr/bin/google-chrome}"
THREADS="${THREADS:-48}"
# wall-clock budget; default 10h = 36000s
DEADLINE_SEC="${DEADLINE_SEC:-36000}"
log "PUPPETEER_EXECUTABLE_PATH=$PUPPETEER_EXECUTABLE_PATH"
log "THREADS=$THREADS"
log "DEADLINE_SEC=$DEADLINE_SEC ($(awk -v s=$DEADLINE_SEC 'BEGIN{printf "%.1fh", s/3600}'))"

mapfile -t POOLS < <(python -c "import config; print('\n'.join(config.data_resource_dirs))")
if [[ ${#POOLS[@]} -eq 0 ]]; then
  log "ERROR: config.data_resource_dirs is empty"
  exit 1
fi
log "POOLS (from config.data_resource_dirs):"
for p in "${POOLS[@]}"; do log "  $p"; done

# -------- 3. rolling rounds --------
START_TS=$(date +%s)
ROUND=0
while :; do
  NOW=$(date +%s)
  ELAPSED=$((NOW - START_TS))
  REMAINING=$((DEADLINE_SEC - ELAPSED))
  if (( REMAINING <= 0 )); then
    log "deadline reached (elapsed=${ELAPSED}s >= ${DEADLINE_SEC}s); not starting new round"
    break
  fi

  ROUND=$((ROUND + 1))
  log "============================================================"
  log "ROUND $ROUND  elapsed=$((ELAPSED/3600))h$(((ELAPSED%3600)/60))m  remaining=$((REMAINING/3600))h$(((REMAINING%3600)/60))m"
  log "disk before round: $(df -h /data | tail -1)"

  for pool in "${POOLS[@]}"; do
    name="$(basename "$pool")"
    out="output/line_pie_donut_chart_only/$name"
    log="logs/10h_run_round_${ROUND}_${name}.log"
    mkdir -p "$out"
    # NOTE: bash 'log' function got shadowed by the loop var 'log' (file path),
    # so use printf directly for this round-line.
    printf '[%s] -- round %d pool=%s out=%s\n' "$(date -Iseconds)" "$ROUND" "$pool" "$out" | tee -a "$SUMMARY"
    python pipeline.py \
      --input  "$pool" \
      --output "$out" \
      --modules infographics_generator \
      --chart-only --output-png \
      --threads "$THREADS" \
      2>&1 | tee "$log" > /dev/null
    printf '[%s] -- round %d pool=%s DONE\n' "$(date -Iseconds)" "$ROUND" "$pool" | tee -a "$SUMMARY"
  done

  printf '[%s] round %d finished. disk after: %s\n' "$(date -Iseconds)" "$ROUND" "$(df -h /data | tail -1)" | tee -a "$SUMMARY"
done

END_TS=$(date +%s)
TOTAL=$((END_TS - START_TS))
printf '[%s] ==== 10h loop done after %dh %dm, completed %d full round(s) ====\n' \
  "$(date -Iseconds)" $((TOTAL/3600)) $(((TOTAL%3600)/60)) "$ROUND" | tee -a "$SUMMARY"