File size: 4,665 Bytes
98c193b 1c9c214 98c193b dbbc6f9 98c193b dbbc6f9 98c193b dbbc6f9 98c193b dbbc6f9 98c193b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | #!/bin/bash
# Runs on the cluster after a grid finishes (SLURM dependency), so results are
# collected, backed up and published even if the laptop is closed, the network
# drops, or the session that launched the work is long gone.
#
# 1. regenerate figures + tables from every run
# 2. back up to HOME (the only nightly-backed-up tier)
# 3. back up weights to SHELL (1 TB, no 90-day purge unlike scratch)
# 4. push to the Hugging Face Hub, with retries
#
#SBATCH -J finalize
#SBATCH -t 03:00:00
#SBATCH -c 4
#SBATCH --mem=16g
#SBATCH -p standard
#SBATCH -o logs/finalize-%j.out
#SBATCH --open-mode=append
source /etc/profile
module load pytorch/2.0.1
USER_SCR=/scratch/zt1/project/msml612/user/$USER
STAR=$USER_SCR/pcmd
export PYTHONPATH=$USER_SCR/pylibs:$PYTHONPATH
# Compute nodes cannot reach HF's Xet CAS endpoint; force the plain LFS path.
export HF_HUB_DISABLE_XET=1
HOME_BK=$HOME/pcmd-backup
SHELL_BK=/afs/shell.umd.edu/project/msml612/user-$USER/msml612-weights
echo "=== 1. collect + plot ==="
cd $STAR/src
RUNS=$STAR/runs OUT=$STAR/figures python collect.py || echo "collect failed (continuing)"
echo "=== 2. back up to HOME (backed up nightly) ==="
mkdir -p $HOME_BK/{results,figures,src,slurm}
cp -f $STAR/figures/* $HOME_BK/figures/ 2>/dev/null
cp -f $STAR/src/*.py $HOME_BK/src/ 2>/dev/null
cp -f $STAR/*.sh $HOME_BK/slurm/ 2>/dev/null
for d in $STAR/runs/*/; do
n=$(basename $d)
[ -f "$d/result.json" ] && cp -f "$d/result.json" "$HOME_BK/results/$n.json"
done
echo " results backed up: $(ls $HOME_BK/results | wc -l)"
echo "=== 3. weights: NOT copied to HOME (quota is 9.5G soft / 19G hard) ==="
# An earlier version copied every model.pt here. That is 15G, which blew through
# both the soft quota and the hard limit, left HOME unwritable, and silently
# broke the Hugging Face push for five consecutive retries - the staging README
# could not be written. Weights stay on scratch; a curated subset goes to the
# Hub in step 4, where there is no such quota.
echo " weights remain on scratch: $(ls -d $STAR/runs/*/model.pt 2>/dev/null | wc -l) checkpoints"
echo " HOME usage now: $(du -sh $HOME_BK 2>/dev/null | cut -f1)"
echo "=== 4. push to Hugging Face (with retries) ==="
# A transient network failure must not mean the results never reach the Hub, so
# retry with backoff rather than giving up on the first error.
for attempt in 1 2 3 4 5; do
python - <<'PY' && break
import os, pathlib, sys
from huggingface_hub import HfApi
tok = pathlib.Path(os.path.expanduser("~/.hf_token")).read_text().strip()
star = os.path.expanduser(f"/scratch/zt1/project/msml612/user/{os.environ['USER']}/pcmd")
stage = pathlib.Path(os.path.expanduser("~/pcmd-backup"))
repo = "GOVINDFROM/parallel-consistent-masked-diffusion"
summary = pathlib.Path(star, "figures", "summary.md")
table = summary.read_text() if summary.exists() else "_(pending)_"
(stage / "README.md").write_text(f"""---
license: apache-2.0
tags: [masked-diffusion, discrete-diffusion, parallel-decoding, text8, from-scratch]
---
# Two Failure Modes of Parallel Decoding in Masked Diffusion
Parallel decoding in masked diffusion loses accuracy for two different reasons
that need opposite fixes:
1. **The marginals are untrained at high mask ratio.** A K-pass decode only ever
evaluates the model at mask ratios {{1, (K-1)/K, ..., 1/K}}, while uniform-t
training spends most of its capacity on nearly-complete states that
high-parallelism decoding never visits. **Fixable only at training time.**
2. **The committed positions carry real mutual information.** Parallel decoding
samples the product of marginals; the truth is the joint. The discarded
dependence is the total correlation of the committed set, upper-bounded by
its summed conditional entropy. **Fixable only at inference, by committing
less.**
Prior work applies inference-time fixes to both, which is why it flatlines on
arithmetic - where mode 2 is identically zero and all the loss is mode 1.
Everything is trained from scratch: no pretrained weights, no pretrained
tokenizer, and the autoregressive evaluator used to score generated text is
itself trained from scratch on the same corpus.
Code: https://github.com/GIND123/MSML612
## Results
{table}
""")
api = HfApi(token=tok)
api.create_repo(repo, repo_type="model", private=True, exist_ok=True)
api.upload_folder(folder_path=str(stage), repo_id=repo, repo_type="model",
commit_message="Automated push from Zaratan")
print("pushed:", f"https://huggingface.co/{repo}")
PY
echo " push attempt $attempt failed; sleeping $((attempt * 60))s"
sleep $((attempt * 60))
done
echo "=== finalize complete ==="
|