#!/usr/bin/env bash # 10-hour rolling loop: re-scan both data pools repeatedly so select_template # naturally rotates through under-used chart_name variations (variation.json # weights each pick by usage frequency, so the same input may end up rendered # with different variations across rounds). # # Stop policy (graceful): before each round, check elapsed wall clock; if >= # DEADLINE_SEC, do NOT start a new round. The currently running round is # always allowed to finish. # # Output: APPENDS into output/line_pie_donut_chart_only// (same dir as # the foundational run). Filenames carry an int timestamp so they never # collide. # # Logs: # logs/10h_run_round__.log per-round per-pool pipeline output # logs/10h_run_summary.log this wrapper's own progress set -u set -o pipefail cd "$(dirname "$0")/.." ROOT="$(pwd)" SUMMARY=logs/10h_run_summary.log mkdir -p logs output/line_pie_donut_chart_only log() { printf '[%s] %s\n' "$(date -Iseconds)" "$*" | tee -a "$SUMMARY"; } log "==== 10h loop launcher ====" log "cwd=$ROOT" # -------- 1. wait for current line_pie_donut session to finish -------- PRIOR_SESSION="${PRIOR_SESSION:-line_pie_donut}" if tmux has-session -t "$PRIOR_SESSION" 2>/dev/null; then log "waiting for prior tmux session '$PRIOR_SESSION' to finish..." while tmux has-session -t "$PRIOR_SESSION" 2>/dev/null; do sleep 60 done log "prior session '$PRIOR_SESSION' is gone, continuing" else log "prior session '$PRIOR_SESSION' not running, starting immediately" fi # -------- 2. config -------- if [[ ! -s allowed_chart_types.json ]]; then log "ERROR: allowed_chart_types.json missing or empty; aborting" exit 1 fi log "whitelist (will be re-read by every worker via mtime cache):" cat allowed_chart_types.json | tee -a "$SUMMARY" export PUPPETEER_EXECUTABLE_PATH="${PUPPETEER_EXECUTABLE_PATH:-/usr/bin/google-chrome}" THREADS="${THREADS:-48}" # wall-clock budget; default 10h = 36000s DEADLINE_SEC="${DEADLINE_SEC:-36000}" log "PUPPETEER_EXECUTABLE_PATH=$PUPPETEER_EXECUTABLE_PATH" log "THREADS=$THREADS" log "DEADLINE_SEC=$DEADLINE_SEC ($(awk -v s=$DEADLINE_SEC 'BEGIN{printf "%.1fh", s/3600}'))" mapfile -t POOLS < <(python -c "import config; print('\n'.join(config.data_resource_dirs))") if [[ ${#POOLS[@]} -eq 0 ]]; then log "ERROR: config.data_resource_dirs is empty" exit 1 fi log "POOLS (from config.data_resource_dirs):" for p in "${POOLS[@]}"; do log " $p"; done # -------- 3. rolling rounds -------- START_TS=$(date +%s) ROUND=0 while :; do NOW=$(date +%s) ELAPSED=$((NOW - START_TS)) REMAINING=$((DEADLINE_SEC - ELAPSED)) if (( REMAINING <= 0 )); then log "deadline reached (elapsed=${ELAPSED}s >= ${DEADLINE_SEC}s); not starting new round" break fi ROUND=$((ROUND + 1)) log "============================================================" log "ROUND $ROUND elapsed=$((ELAPSED/3600))h$(((ELAPSED%3600)/60))m remaining=$((REMAINING/3600))h$(((REMAINING%3600)/60))m" log "disk before round: $(df -h /data | tail -1)" for pool in "${POOLS[@]}"; do name="$(basename "$pool")" out="output/line_pie_donut_chart_only/$name" log="logs/10h_run_round_${ROUND}_${name}.log" mkdir -p "$out" # NOTE: bash 'log' function got shadowed by the loop var 'log' (file path), # so use printf directly for this round-line. printf '[%s] -- round %d pool=%s out=%s\n' "$(date -Iseconds)" "$ROUND" "$pool" "$out" | tee -a "$SUMMARY" python pipeline.py \ --input "$pool" \ --output "$out" \ --modules infographics_generator \ --chart-only --output-png \ --threads "$THREADS" \ 2>&1 | tee "$log" > /dev/null printf '[%s] -- round %d pool=%s DONE\n' "$(date -Iseconds)" "$ROUND" "$pool" | tee -a "$SUMMARY" done printf '[%s] round %d finished. disk after: %s\n' "$(date -Iseconds)" "$ROUND" "$(df -h /data | tail -1)" | tee -a "$SUMMARY" done END_TS=$(date +%s) TOTAL=$((END_TS - START_TS)) printf '[%s] ==== 10h loop done after %dh %dm, completed %d full round(s) ====\n' \ "$(date -Iseconds)" $((TOTAL/3600)) $(((TOTAL%3600)/60)) "$ROUND" | tee -a "$SUMMARY"