Buckets:
| # Full sync of the project to the PUBLIC bucket. Keys live in /workspace/.cache (outside the project) and are never read here. | |
| set -u | |
| cd /workspace/decision-reasoner && source .venv/bin/activate | |
| B=hf://buckets/Mercity/Desicion-Model-Custom | |
| SP=/tmp/claude-0/-workspace/8842c99f-448c-4cba-80ab-3e9c26b892df/scratchpad | |
| t() { echo "$(date -u +%T) $*"; } | |
| # ---- code: the project's own files (no venv, data, results, logs, checkpoints), plus the overnight driver scripts | |
| rm -rf $SP/bucket_code && mkdir -p $SP/bucket_code/ops | |
| rsync -a --exclude .venv --exclude data --exclude results --exclude logs --exclude checkpoints --exclude markers \ | |
| --exclude '__pycache__' --exclude 'runs/*/data' --exclude 'runs/*/results' --exclude 'runs/*/checkpoints' --exclude 'runs/*/markers' \ | |
| --exclude 'runs/*/.venv' --exclude data_qwen3 --exclude 'dl_*.log' --exclude install.log --no-links ./ $SP/bucket_code/ 2>/dev/null | |
| cp results/DEVIATIONS.md results/NOTEBOOK.md results/report.md results/FINAL_REPORT.md $SP/bucket_code/ 2>/dev/null | |
| cp $SP/*.sh $SP/watch_*.py $SP/quick_eval.py $SP/oom_test.py $SP/jsdomtest/*.js $SP/bucket_code/ops/ 2>/dev/null | |
| rm -f $SP/bucket_code/ops/bucket_sync_full.sh.tmp | |
| # ---- refuse to sync if anything that looks like a key or token is in the files about to go up | |
| if grep -rIlE "hf_[A-Za-z0-9]{30,}|sk-or-v1-[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{40,}" --exclude-dir=.venv --exclude-dir=published --exclude-dir=dataset \ | |
| --exclude-dir=checkpoints . $SP/bucket_code 2>/dev/null | grep -v "^./data/raw/" | head -5 | grep . ; then t "KEY-LIKE STRING FOUND, not syncing"; exit 1; fi | |
| t scan-clean | |
| t code; hf buckets sync $SP/bucket_code $B/code --quiet | |
| # ---- v1 (run1): results incl. raw chains, data (not the raw downloads), logs other than V2 / RL / 2B / Jevals | |
| t run1-results; hf buckets sync results $B/run1/results --quiet | |
| t run1-data; hf buckets sync data $B/run1/data --exclude "raw/*" --exclude "jevals/*" --exclude "s1mb/*" --quiet | |
| t run1-extra; hf buckets sync data_qwen3 $B/run1/data_qwen3 --quiet | |
| for f in dl_*.log install.log; do [ -f $f ] && hf buckets cp $f $B/run1/setup_logs/$f --quiet; done | |
| t run1-logs; hf buckets sync logs $B/run1/logs --exclude "v2_*" --exclude "v2b_*" --exclude "rl_*" --exclude "jevals_*" --exclude "after_2b*" --exclude "bucket_*" --exclude "s1mb_*" --exclude "skipped_runs*" --quiet | |
| # ---- V2 + RL (run2_v2): results incl. raw chains and adapters, data, logs, RL checkpoints (adapters, optimizer, chains), merged models | |
| t run2-results; hf buckets sync runs/v2/results $B/run2_v2/results --quiet | |
| t run2-data; hf buckets sync runs/v2/data $B/run2_v2/data --exclude "raw/*" --quiet | |
| t run2-logs; hf buckets sync logs $B/run2_v2/logs --include "v2_*" --include "rl_*" --include "after_2b*" --exclude "*" --quiet | |
| t run2-rl-ckpts; hf buckets sync runs/v2/checkpoints/rl_grpo $B/run2_v2/checkpoints/rl_grpo --quiet | |
| t run2-rl-ckpts-b; hf buckets sync runs/v2/checkpoints/rl_grpo_b $B/run2_v2/checkpoints/rl_grpo_b --quiet | |
| for c in v2_joint_merged rl60_merged; do t "run2-merged $c"; hf buckets sync runs/v2/checkpoints/$c $B/run2_v2/checkpoints/$c --quiet; done | |
| # ---- Jevals: suite files, rebuilt items, our runs, the score table | |
| t jevals; hf buckets sync data/jevals $B/jevals/data --quiet | |
| hf buckets cp results/jevals.json $B/jevals/jevals.json --quiet | |
| for f in logs/jevals_*.log; do hf buckets cp $f $B/jevals/logs/$(basename $f) --quiet; done | |
| # ---- 2B (run3_2b): whatever exists so far (run_2b.sh keeps syncing results and logs as it goes) | |
| t run3; hf buckets sync runs/v2_2b/results $B/run3_2b/results --quiet | |
| hf buckets sync logs $B/run3_2b/logs --include "v2b_*" --exclude "*" --quiet | |
| hf buckets sync runs/v2_2b/checkpoints/v2b_ro_merged $B/run3_2b/checkpoints/v2b_ro_merged --quiet | |
| # ---- S1MB: our items, raw readouts per model, the score table, logs (their dataset and published results stay at hotchpotch/*) | |
| t s1mb; hf buckets sync data/s1mb/runs $B/s1mb/runs --quiet | |
| for f in data/s1mb/items.jsonl data/s1mb/english-v1.json; do hf buckets cp $f $B/s1mb/data/$(basename $f) --quiet; done | |
| hf buckets sync data/s1mb/benchmarks $B/s1mb/data/benchmarks --quiet | |
| hf buckets cp results/s1mb.json $B/s1mb/s1mb.json --quiet | |
| for f in logs/s1mb_*.log logs/skipped_runs.log; do [ -f $f ] && hf buckets cp $f $B/s1mb/logs/$(basename $f) --quiet; done | |
| hf buckets sync data/s1mb/think $B/s1mb/think --quiet | |
| # ---- the full report (PDF, its markdown source and figures) and the analysis outputs behind it | |
| t report; hf buckets sync results/report $B/report --quiet | |
| hf buckets sync results/report_figs $B/report/figs --quiet | |
| for f in results/final_stats.json results/final_stats.md results/examples_analysis.json results/examples_analysis.md results/yesno_offset.json; do | |
| hf buckets cp $f $B/analysis/$(basename $f) --quiet; done | |
| for f in logs/s1mb_routed.log; do [ -f $f ] && hf buckets cp $f $B/s1mb/logs/$(basename $f) --quiet; done | |
| # ---- top level: README and the final report | |
| hf buckets cp $SP/bucket_README.md $B/README.md --quiet | |
| hf buckets cp results/FINAL_REPORT.md $B/FINAL_REPORT.md --quiet | |
| # ---- the dashboard page as published | |
| t dashboard; hf buckets cp $SP/decision_reasoner_dashboard.html $B/dashboard/index.html --quiet | |
| hf buckets cp $SP/decision_reasoner_dashboard.html.json $B/dashboard/data.json --quiet | |
| t DONE | |
Xet Storage Details
- Size:
- 5.42 kB
- Xet hash:
- 66e341c536d627eea48d830f5c60576cbfe4c6be9f766e817931f8724fa69596
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.