File size: 5,565 Bytes
ec0a9aa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | #!/usr/bin/env bash
# WorldCache threshold benchmark script
# Tests rel_l1_thresh at 0.03, 0.05, 0.08, 0.10 with only 3 samples each (single chunk)
# for quick wall-time and skip-ratio comparison.
#
# Usage (from video_gen_physics root):
# bash bench_worldcache_thresh.sh 2>&1 | tee output/worldcache_bench/bench.log
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${SCRIPT_DIR}"
source ./models/DreamDojo/.venv/bin/activate
export PYTHONPATH="./models/DreamDojo:${PYTHONPATH:-}"
CKPT_DIR="./checkpoints/dreamdojo/2B_GR1_post-train"
DATASET="./checkpoints/dreamdojo/datasets/PhysicalAI-Robotics-GR00T-Teleop-GR1/GR1_robot"
EXPERIMENT="dreamdojo_2b_480_640_gr1"
OUT_BASE="./output/worldcache_bench"
NUM_SAMPLES=3 # 3 samples for quick wall-time comparison
NUM_FRAMES=37 # default chunk size; 1 long sequence per sample
mkdir -p "${OUT_BASE}"
# -----------------------------------------------------------------------
# Helper: run one configuration and capture key metrics
# -----------------------------------------------------------------------
run_config() {
local label="$1" # e.g. "thresh_0.03"
local extra_args="$2" # worldcache CLI args
local out_dir="${OUT_BASE}/${label}"
mkdir -p "${out_dir}"
echo ""
echo "========================================================"
echo " Starting: ${label}"
echo " Time: $(date -u +'%Y-%m-%dT%H:%M:%SZ')"
echo "========================================================"
local start_s
start_s=$(date +%s)
# shellcheck disable=SC2086
python -m models.DreamDojo.examples.action_conditioned \
-o "${out_dir}/logs" \
--checkpoints-dir "${CKPT_DIR}" \
--experiment "${EXPERIMENT}" \
--save-dir "${out_dir}" \
--num-frames "${NUM_FRAMES}" \
--num-samples "${NUM_SAMPLES}" \
--dataset-path "${DATASET}" \
--data-split test \
--deterministic-uniform-sampling \
${extra_args} \
2>&1 | tee "${out_dir}/run.log"
local end_s
end_s=$(date +%s)
local elapsed=$(( end_s - start_s ))
echo ""
echo "[BENCH] ${label}: done in ${elapsed}s"
# Extract all WorldCache skip ratios
local skip_line
skip_line=$(grep -oE '\[WorldCache\] Skipped [0-9]+/[0-9]+ \([0-9.]+%\)' "${out_dir}/run.log" | tail -n 5 | tr '\n' ' ' || true)
if [[ -n "${skip_line}" ]]; then
echo "[BENCH] ${label}: skip_ratios => ${skip_line}"
else
echo "[BENCH] ${label}: (no WorldCache skip ratio — baseline or error)"
skip_line="N/A"
fi
# Extract it/s from tqdm progress bar (last 5 readings)
local speed_line
speed_line=$(grep -oE '[0-9]+\.[0-9]+it/s' "${out_dir}/run.log" | tail -n 5 | tr '\n' ' ' || true)
if [[ -n "${speed_line}" ]]; then
echo "[BENCH] ${label}: last it/s => ${speed_line}"
else
speed_line="N/A"
fi
# Summary line
local summary="${label} elapsed=${elapsed}s skip=[${skip_line}] speed=[${speed_line}]"
echo "${summary}"
echo "${summary}" >> "${OUT_BASE}/summary.txt"
echo "---"
}
# -----------------------------------------------------------------------
# Clear previous summary
# -----------------------------------------------------------------------
{
echo "# WorldCache threshold benchmark"
echo "# Run at: $(date -u +'%Y-%m-%dT%H:%M:%SZ')"
echo "# NUM_SAMPLES=${NUM_SAMPLES} NUM_FRAMES=${NUM_FRAMES}"
echo ""
} > "${OUT_BASE}/summary.txt"
# -----------------------------------------------------------------------
# 1. BASELINE — dense, no cache
# -----------------------------------------------------------------------
run_config "baseline" ""
# -----------------------------------------------------------------------
# 2. thresh=0.03 (original CLI value — now expected to be too strict)
# -----------------------------------------------------------------------
run_config "thresh_0.03" "\
--worldcache-enabled \
--worldcache-num-steps 35 \
--worldcache-rel-l1-thresh 0.03 \
--worldcache-probe-depth 4 \
--worldcache-ret-ratio 0.5"
# -----------------------------------------------------------------------
# 3. thresh=0.05
# -----------------------------------------------------------------------
run_config "thresh_0.05" "\
--worldcache-enabled \
--worldcache-num-steps 35 \
--worldcache-rel-l1-thresh 0.05 \
--worldcache-probe-depth 4 \
--worldcache-ret-ratio 0.5"
# -----------------------------------------------------------------------
# 4. thresh=0.08 (primary candidate for corrected normalization)
# -----------------------------------------------------------------------
run_config "thresh_0.08" "\
--worldcache-enabled \
--worldcache-num-steps 35 \
--worldcache-rel-l1-thresh 0.08 \
--worldcache-probe-depth 4 \
--worldcache-ret-ratio 0.5"
# -----------------------------------------------------------------------
# 5. thresh=0.10
# -----------------------------------------------------------------------
run_config "thresh_0.10" "\
--worldcache-enabled \
--worldcache-num-steps 35 \
--worldcache-rel-l1-thresh 0.10 \
--worldcache-probe-depth 4 \
--worldcache-ret-ratio 0.5"
# -----------------------------------------------------------------------
# Final summary
# -----------------------------------------------------------------------
echo ""
echo "========================================================"
echo " BENCHMARK COMPLETE — SUMMARY"
echo "========================================================"
cat "${OUT_BASE}/summary.txt"
|