H2-conv (3.17) step 12000: raw + EMA export

image_video_conv/step12000/README.md

@@ -0,0 +1,11 @@
+# image_video_conv / step 12000 (H2-conv, grain_prior 3.17)
+
+Intermediate checkpoint of the conv-decoder run (d_f 52, 16,420 numbers per 256x256 image; SD-VAE style conv decoder
+after the splat, splat_direct, 8-head mid attention, conv_low_in). Trained on image + video, 8 GPUs, batch 384 images +
+96 clips per step, warm start from grain-prior-g2df-20k; WSD not yet in its decay phase at this step.
+
+- `raw/`: training weights at step 12000 (`grain_prior.pt`, `config.json`), DINOv3 excluded (load with your own copy).
+- `ema/`: EMA 0.999 at step 12000 (still lagging at this step: use `raw/`).
+- `s12000_*.md`: eval tables (200 ImageNet val images; 20 OpenVid held-out clips, stride 8 and 2).
+- `launcher.sh`: the training flags (paths removed).
+

image_video_conv/step12000/ema/config.json

@@ -0,0 +1,248 @@
+{
+ "step": 12000,
+ "n_params": 294408173,
+ "cfg": {
+  "image_size": 256,
+  "patch": 16,
+  "dino_repo": "facebook/dinov3-vitl16-pretrain-lvd1689m",
+  "dino_dim": 1024,
+  "dino_registers": 4,
+  "dino_layers": [
+   8,
+   16,
+   24
+  ],
+  "stub_dino_dim": 64,
+  "stub_dino_blocks": 4,
+  "pix_dim": 256,
+  "dim": 512,
+  "heads": 8,
+  "layers": 6,
+  "mlp_ratio": 4.0,
+  "rope_dims": [
+   16,
+   24,
+   24
+  ],
+  "rope_base": 100.0,
+  "shared_streams": false,
+  "keyframe_every": 4,
+  "d_c_image": 47,
+  "d_c_video": 23,
+  "fov_min": 30.0,
+  "fov_max": 120.0,
+  "fov_init": 50.0,
+  "rho_refine_heads": 4,
+  "flow_grid": 4,
+  "d_r": 24,
+  "l_max": 0,
+  "budget": 12288,
+  "p_l0": 0.15,
+  "enc_r_layers": 4,
+  "dec_dim": 512,
+  "dec_heads": 8,
+  "knn": 9,
+  "softmin_beta": 1.0,
+  "cut_eps": 0.0001,
+  "z_eps": 0.01,
+  "backface_kappa": 10.0,
+  "dec_pred_blocks": 2,
+  "dec_innov_blocks": 2,
+  "prev_read": true,
+  "n_freq": 6,
+  "conv_channels": [
+   512,
+   512,
+   128
+  ],
+  "conv_res_blocks": 1,
+  "w_rgb": 1.0,
+  "w_feat": 1.0,
+  "w_geo": 0.5,
+  "w_pred": 0.5,
+  "geo_rgb": true,
+  "w_flow": 0.01,
+  "w_smooth": 0.01,
+  "w_kl_c": 1e-06,
+  "w_kl_d": 1e-06,
+  "w_link": 0.25,
+  "depth_probe": null,
+  "w_depth_prior": 0.0,
+  "depth_prior_kind": "silog",
+  "depth_prior_on": [
+   "image",
+   "video"
+  ],
+  "w_depth_gt": 0.0,
+  "depth_gt_lam": 0.85,
+  "w_rho_distill": 0.0,
+  "w_lpips": 0.2,
+  "lpips_max_frames": 32,
+  "charbonnier_eps": 0.001,
+  "detach_rho_image": true,
+  "extra": {},
+  "stem_channels": [
+   64,
+   128,
+   256,
+   512
+  ],
+  "pe_freq": 8,
+  "mlp_hidden": 512,
+  "stratum": 1,
+  "d_f": 52,
+  "coarse_grid": 4,
+  "d_coarse": 128,
+  "grain_blocks": 2,
+  "grain_read_s": 1.0,
+  "sigma_min": 0.15,
+  "sigma_span": 1.1,
+  "rho_lo": -7.0,
+  "rho_hi": 5.0,
+  "temp_s": 1.0,
+  "n_motion": 16,
+  "d_b": 2,
+  "n_slots": 52,
+  "d_e": 20,
+  "queue": 3,
+  "fold_mode": "memory",
+  "nested_L": [
+   0,
+   16,
+   32,
+   52,
+   52,
+   52,
+   52
+  ],
+  "image_fov_fixed": true,
+  "pp_range": 0.05,
+  "refine_passes": 2,
+  "upd_blocks": 2,
+  "corr_eps": 0.05,
+  "corr_s": [
+   3.0,
+   1.5
+  ],
+  "frame_seconds": 0.125,
+  "res_layers": 4,
+  "read_layers": 2,
+  "tau_d": 0.1,
+  "tau_d_mode": "median",
+  "rgb_channels": [
+   256,
+   128,
+   64,
+   32
+  ],
+  "rgb_from": "hidden",
+  "query_grid": 64,
+  "read_qk_norm": false,
+  "readout_version": "legacy",
+  "read_strict": true,
+  "read_coarse_init": true,
+  "read_sink": true,
+  "read_radius_scale": true,
+  "read_radius_clip": 8.0,
+  "innov_stopgrad_base": true,
+  "w_geom_dense": 1.0,
+  "w_geom_ramp_steps": 0,
+  "dino_multi_layer": true,
+  "read_coarse_kv": true,
+  "coarse_sink": true,
+  "read_relpos_value": true,
+  "relpos_rank": 8,
+  "rgb_coarse_start": true,
+  "w_rgb_coarse": 1.0,
+  "rgb_head": "conv",
+  "rgb_pw_mlp_ratio": 4.0,
+  "rgb_in_feat": false,
+  "read_split_layer2": true,
+  "read_prev_frame": true,
+  "w_pred_frame": 0.5,
+  "sem_target": "",
+  "w_sem": 0.0,
+  "sem_dim": 768,
+  "w_gan": 0.0,
+  "gan_start": 0,
+  "disc_channels": 64,
+  "disc_layers": 3,
+  "anchor_df": false,
+  "train_scope": "all",
+  "rgb_ctx_blocks": 0,
+  "rgb_decoder": "splat",
+  "splat_layers": 8,
+  "splat_feat": 128,
+  "splat_atoms": 12,
+  "splat_window": 2,
+  "splat_wmax": 8.0,
+  "splat_index": "projected",
+  "splat_cap": 4,
+  "splat_pix_feat": 0,
+  "splat_pix_atoms": 6,
+  "splat_pix_wmax": 8.0,
+  "rgb_pix_blocks": 0,
+  "rgb_pix_width": 128,
+  "splat_dim": 768,
+  "conv_blocks": [
+   4,
+   4,
+   3
+  ],
+  "conv_mid_attn": true,
+  "conv_act_ckpt": true,
+  "conv_attn_heads": 8,
+  "conv_low_in": true,
+  "splat_direct": true,
+  "read_normalizer": "softmax",
+  "read_topk": 0,
+  "entmax_alpha_init": 1.0,
+  "alpha_anneal_steps": 0,
+  "state_noise_bits": 0,
+  "geom_noise_px": 0.0,
+  "sigma_param": "log",
+  "sigma_lo": 0.02,
+  "query_version": 1,
+  "measurement_version": 1,
+  "render_profile": "uniform_midpoint_v1",
+  "innovation_semantics": "legacy_image",
+  "render_samples": 32,
+  "query_chunk": 2048,
+  "read_act_ckpt": false,
+  "render_near": 0.1,
+  "render_far": 4.0,
+  "background_rgb": [
+   0.0,
+   0.0,
+   0.0
+  ],
+  "prior_layers": 6,
+  "horizons": [
+   1,
+   4
+  ],
+  "query_init": "cond",
+  "geom_bias": true,
+  "memory": true,
+  "phase": "codec",
+  "world_given_p": 0.5,
+  "w_edge": 0.5,
+  "feat_sg_f": true,
+  "w_logn": 0.01,
+  "w_base": 0.5,
+  "w_init": 0.25,
+  "w_geom": 1.0,
+  "w_pose": 0.1,
+  "w_motion": 0,
+  "w_warp": 0.1,
+  "w_warp_flow": 1.0,
+  "warp_radius": 3,
+  "warp_tau": 0.01,
+  "grain_df_k": 8,
+  "w_code": 0.0001,
+  "depth_delta": 0.1,
+  "depth_loss": "huber",
+  "depth_bg_far": 0.0
+ },
+ "excluded": "stems.dino.model.*"
+}

image_video_conv/step12000/ema/grain_prior.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d38633f27a6ede6e3ec19b52323bd432f6432d44de0d7f319150bbfab1222e9
+size 1178087048

image_video_conv/step12000/launcher.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# H2-conv: 3.17 (review fixes) on 8 GPUs. Fresh start from G2df-iv 20k EMA: d_f 52 (state 16 420), splat 8 blocks @768
+# feat 128 / 12 atoms + splat_direct, conv decoder 512/512/128 x 4/4/3 with 8-head mid attention and the coarse
+# image as input (conv_low_in), activation checkpointing. DINO frozen, everything else trains; tensors whose shape
+# changed with d_f keep their fresh init (--init-shape-ok). Same data / losses / schedule as G2df-iv (image 0.3 /
+# video 0.7, WSD), ckpt every 250. Batch 64 images + 16 clips per GPU (512 + 128 global). Stop: touch STOP_H2.
+P=<path> D=<path> O=$P/H2-conv; mkdir -p $O
+INIT_CKPT=$P/G2df-iv/step_0020000_ema.pt
+CODE=<path> cd $CODE
+S=${STEPS:-20000}; MEM_MAX=${MEM_MAX:-250}; GPUS=${GPUS:-0,1,2,3,4,5,6,7}; NGPU=$(echo $GPUS | tr ',' '\n' | wc -l)
+IMG="--imagenet $D/imagenet-1k"
+VID="--openvid $D/dino_v0_data/openvid1m/manifest.jsonl --ssv2-train $D/dino_v0_data/ssv2/ssv2_train_manifest.jsonl --ssv2-weight 0.3 --frame-stride 2,4,8"
+BATCHES=${BATCHES:-64+16,48+12,32+8}
+SETS="--set d_f=52 --set rgb_decoder=splat --set feat_sg_f=true --set w_lpips=0.2 --set w_edge=0.5 --set read_act_ckpt=false --set lpips_max_frames=32 --set query_chunk=2048"
+GEO="--set w_warp=0.1 --set w_motion=0 --set splat_index=projected --set splat_cap=4 --set w_warp_flow=1.0 --set warp_radius=3 --set warp_tau=0.01 --set grain_df_k=8"
+NEW="--set splat_dim=768 --set splat_layers=8 --set splat_feat=128 --set splat_atoms=12 --set splat_direct=true --set rgb_head=conv --set conv_channels=[512,512,128] --set conv_blocks=[4,4,3] --set conv_attn_heads=8 --set conv_low_in=true --set conv_act_ckpt=true"
+export TORCHINDUCTOR_CACHE_DIR=<path> TRITON_CACHE_DIR=<path>
+mkdir -p $TRITON_CACHE_DIR
+export TORCHINDUCTOR_MAX_AUTOTUNE=${MAXAT:-1} TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=${MAXAT:-1}
+. <path>
+gpu_busy() {  # more than 5 % VRAM in use on any of our GPUs
+  local out v; out=$(rocm-smi --showmemuse 2>/dev/null)
+  for g in ${GPUS//,/ }; do v=$(echo "$out" | grep "GPU\[$g\]" | grep "VRAM%" | awk '{print $NF}'); [ "${v:-0}" -gt 5 ] && return 0; done
+  return 1
+}
+echo "waiting for GPU $GPUS $(date -u)" >> $O/launch.txt
+while gpu_busy; do [ -e $P/STOP_H2 ] && exit 0; sleep 60; done
+echo "GPU $GPUS free, starting $(date -u)" >> $O/launch.txt
+train() {  # <out> <steps> <extra...>
+  local out=$1 steps=$2; shift 2
+  gpuardian run --name muse-spark-mavt -- /usr/bin/env -u KEY HIP_VISIBLE_DEVICES=$GPUS PYTHONFAULTHANDLER=1 PYTHONPATH=src HF_HUB_OFFLINE=1 OMP_NUM_THREADS=16 \
+    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+    <path> -m torch.distributed.run --standalone --nproc_per_node $NGPU --max-restarts=0 \
+    scripts/train_grain_prior.py --config configs/grain_prior/final.yaml \
+    --set dino_repo=<path> $SETS $GEO $NEW --set "nested_L=[0,16,32,52,52,52,52]" \
+    $IMG $VID $BATCH --modality-loss-weights image:0.3,video:0.7 --num-workers 24 \
+    --compile-regions ${REGIONS:-Block,ReadLayer} --steps $steps --decay-steps $((steps / 10)) --ckpt-every 250 --out $out "$@" >> $out/train.log 2>&1
+}
+bat() { local b=$1; echo "--batch-image ${b%+*} --batch-video ${b#*+}"; }
+if [ -f $O/last.pt ]; then INIT="--resume $O/last.pt"; CK=$O/last.pt; else INIT="--init $INIT_CKPT --resume none --init-new-ok --init-shape-ok"; CK=$INIT_CKPT; fi
+if [ ! -f $O/probe/batch.txt ]; then
+  mkdir -p $O/probe
+  for b in ${BATCHES//,/ }; do
+    BATCH=$(bat $b)
+    : > $O/probe/log.jsonl; rm -f $O/probe/last*.pt
+    train $O/probe 40 --init $CK --resume none --init-new-ok --init-shape-ok --log-every 10 --ckpt-every 1000000; R=$?
+    PEAK=$(python3 -c "import json;print(max(json.loads(l).get(\"peak_mem_gb\",0) for l in open(\"$O/probe/log.jsonl\")))" 2>/dev/null || echo 999)
+    echo "probe ${NGPU}gpu $BATCH exit=$R peak_mem_gb=$PEAK $(date -u)" >> $O/probe.txt
+    if [ $R -eq 0 ] && python3 -c "import sys;sys.exit(1 if float(\"$PEAK\")>$MEM_MAX else 0)"; then echo "${NGPU}gpu $BATCH" > $O/probe/batch.txt; break; fi
+  done
+  [ -f $O/probe/batch.txt ] || { echo "no batch fits under MEM_MAX: not starting" >> $O/probe.txt; exit 1; }
+fi
+BATCH=$(cut -d" " -f2- $O/probe/batch.txt)
+[ -e $P/STOP_H2 ] && exit 0
+try=0
+while [ ! -e $P/STOP_H2 ]; do
+  train $O $S $INIT; R=$?
+  echo "EXIT=$R try=$try $(date -u)" >> $O/train.log
+  if [ $R -eq 0 ] || [ ! -f $O/last.pt ]; then break; fi
+  [ $try -ge 3 ] && { echo "giving up after $try retries" >> $O/train.log; break; }
+  INIT="--resume $O/last.pt"; try=$((try + 1)); sleep 30
+done
+echo "LAUNCHER_EXIT=$R $(date -u)" >> $O/launch.txt

image_video_conv/step12000/raw/config.json

@@ -0,0 +1,248 @@
+{
+ "step": 12000,
+ "n_params": 294408173,
+ "cfg": {
+  "image_size": 256,
+  "patch": 16,
+  "dino_repo": "facebook/dinov3-vitl16-pretrain-lvd1689m",
+  "dino_dim": 1024,
+  "dino_registers": 4,
+  "dino_layers": [
+   8,
+   16,
+   24
+  ],
+  "stub_dino_dim": 64,
+  "stub_dino_blocks": 4,
+  "pix_dim": 256,
+  "dim": 512,
+  "heads": 8,
+  "layers": 6,
+  "mlp_ratio": 4.0,
+  "rope_dims": [
+   16,
+   24,
+   24
+  ],
+  "rope_base": 100.0,
+  "shared_streams": false,
+  "keyframe_every": 4,
+  "d_c_image": 47,
+  "d_c_video": 23,
+  "fov_min": 30.0,
+  "fov_max": 120.0,
+  "fov_init": 50.0,
+  "rho_refine_heads": 4,
+  "flow_grid": 4,
+  "d_r": 24,
+  "l_max": 0,
+  "budget": 12288,
+  "p_l0": 0.15,
+  "enc_r_layers": 4,
+  "dec_dim": 512,
+  "dec_heads": 8,
+  "knn": 9,
+  "softmin_beta": 1.0,
+  "cut_eps": 0.0001,
+  "z_eps": 0.01,
+  "backface_kappa": 10.0,
+  "dec_pred_blocks": 2,
+  "dec_innov_blocks": 2,
+  "prev_read": true,
+  "n_freq": 6,
+  "conv_channels": [
+   512,
+   512,
+   128
+  ],
+  "conv_res_blocks": 1,
+  "w_rgb": 1.0,
+  "w_feat": 1.0,
+  "w_geo": 0.5,
+  "w_pred": 0.5,
+  "geo_rgb": true,
+  "w_flow": 0.01,
+  "w_smooth": 0.01,
+  "w_kl_c": 1e-06,
+  "w_kl_d": 1e-06,
+  "w_link": 0.25,
+  "depth_probe": null,
+  "w_depth_prior": 0.0,
+  "depth_prior_kind": "silog",
+  "depth_prior_on": [
+   "image",
+   "video"
+  ],
+  "w_depth_gt": 0.0,
+  "depth_gt_lam": 0.85,
+  "w_rho_distill": 0.0,
+  "w_lpips": 0.2,
+  "lpips_max_frames": 32,
+  "charbonnier_eps": 0.001,
+  "detach_rho_image": true,
+  "extra": {},
+  "stem_channels": [
+   64,
+   128,
+   256,
+   512
+  ],
+  "pe_freq": 8,
+  "mlp_hidden": 512,
+  "stratum": 1,
+  "d_f": 52,
+  "coarse_grid": 4,
+  "d_coarse": 128,
+  "grain_blocks": 2,
+  "grain_read_s": 1.0,
+  "sigma_min": 0.15,
+  "sigma_span": 1.1,
+  "rho_lo": -7.0,
+  "rho_hi": 5.0,
+  "temp_s": 1.0,
+  "n_motion": 16,
+  "d_b": 2,
+  "n_slots": 52,
+  "d_e": 20,
+  "queue": 3,
+  "fold_mode": "memory",
+  "nested_L": [
+   0,
+   16,
+   32,
+   52,
+   52,
+   52,
+   52
+  ],
+  "image_fov_fixed": true,
+  "pp_range": 0.05,
+  "refine_passes": 2,
+  "upd_blocks": 2,
+  "corr_eps": 0.05,
+  "corr_s": [
+   3.0,
+   1.5
+  ],
+  "frame_seconds": 0.125,
+  "res_layers": 4,
+  "read_layers": 2,
+  "tau_d": 0.1,
+  "tau_d_mode": "median",
+  "rgb_channels": [
+   256,
+   128,
+   64,
+   32
+  ],
+  "rgb_from": "hidden",
+  "query_grid": 64,
+  "read_qk_norm": false,
+  "readout_version": "legacy",
+  "read_strict": true,
+  "read_coarse_init": true,
+  "read_sink": true,
+  "read_radius_scale": true,
+  "read_radius_clip": 8.0,
+  "innov_stopgrad_base": true,
+  "w_geom_dense": 1.0,
+  "w_geom_ramp_steps": 0,
+  "dino_multi_layer": true,
+  "read_coarse_kv": true,
+  "coarse_sink": true,
+  "read_relpos_value": true,
+  "relpos_rank": 8,
+  "rgb_coarse_start": true,
+  "w_rgb_coarse": 1.0,
+  "rgb_head": "conv",
+  "rgb_pw_mlp_ratio": 4.0,
+  "rgb_in_feat": false,
+  "read_split_layer2": true,
+  "read_prev_frame": true,
+  "w_pred_frame": 0.5,
+  "sem_target": "",
+  "w_sem": 0.0,
+  "sem_dim": 768,
+  "w_gan": 0.0,
+  "gan_start": 0,
+  "disc_channels": 64,
+  "disc_layers": 3,
+  "anchor_df": false,
+  "train_scope": "all",
+  "rgb_ctx_blocks": 0,
+  "rgb_decoder": "splat",
+  "splat_layers": 8,
+  "splat_feat": 128,
+  "splat_atoms": 12,
+  "splat_window": 2,
+  "splat_wmax": 8.0,
+  "splat_index": "projected",
+  "splat_cap": 4,
+  "splat_pix_feat": 0,
+  "splat_pix_atoms": 6,
+  "splat_pix_wmax": 8.0,
+  "rgb_pix_blocks": 0,
+  "rgb_pix_width": 128,
+  "splat_dim": 768,
+  "conv_blocks": [
+   4,
+   4,
+   3
+  ],
+  "conv_mid_attn": true,
+  "conv_act_ckpt": true,
+  "conv_attn_heads": 8,
+  "conv_low_in": true,
+  "splat_direct": true,
+  "read_normalizer": "softmax",
+  "read_topk": 0,
+  "entmax_alpha_init": 1.0,
+  "alpha_anneal_steps": 0,
+  "state_noise_bits": 0,
+  "geom_noise_px": 0.0,
+  "sigma_param": "log",
+  "sigma_lo": 0.02,
+  "query_version": 1,
+  "measurement_version": 1,
+  "render_profile": "uniform_midpoint_v1",
+  "innovation_semantics": "legacy_image",
+  "render_samples": 32,
+  "query_chunk": 2048,
+  "read_act_ckpt": false,
+  "render_near": 0.1,
+  "render_far": 4.0,
+  "background_rgb": [
+   0.0,
+   0.0,
+   0.0
+  ],
+  "prior_layers": 6,
+  "horizons": [
+   1,
+   4
+  ],
+  "query_init": "cond",
+  "geom_bias": true,
+  "memory": true,
+  "phase": "codec",
+  "world_given_p": 0.5,
+  "w_edge": 0.5,
+  "feat_sg_f": true,
+  "w_logn": 0.01,
+  "w_base": 0.5,
+  "w_init": 0.25,
+  "w_geom": 1.0,
+  "w_pose": 0.1,
+  "w_motion": 0,
+  "w_warp": 0.1,
+  "w_warp_flow": 1.0,
+  "warp_radius": 3,
+  "warp_tau": 0.01,
+  "grain_df_k": 8,
+  "w_code": 0.0001,
+  "depth_delta": 0.1,
+  "depth_loss": "huber",
+  "depth_bg_far": 0.0
+ },
+ "excluded": "stems.dino.model.*"
+}

image_video_conv/step12000/raw/grain_prior.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:885449bb56d989b7f83bb2cc64fb3004ed3ceaa0b1b4a7c5057420a3b61903f9
+size 1178087048