Upload folder using huggingface_hub
Browse files- online/A_control/offline_baseline_stage1_ga32.yaml +38 -0
- online/A_control/online_baseline_stage1_ga32.yaml +54 -0
- online/B_proposeinit/offline_baseline_stage1_ga32_proposeinit.yaml +24 -0
- online/B_proposeinit/online_baseline_stage1_ga32_proposeinit.yaml +32 -0
- online/C_hardregion/offline_baseline_stage1_ga32_hardregion.yaml +36 -0
- online/C_hardregion/online_baseline_stage1_ga32_hardregion.yaml +52 -0
- online/Cplus_hardregion_beta/offline_baseline_stage1_ga32_hardregion_beta.yaml +33 -0
- online/Cplus_hardregion_beta/online_baseline_stage1_ga32_hardregion_beta.yaml +29 -0
- online/Cplus_hardregion_beta/online_model_0009999_early_aq0.16.pth +3 -0
- online/Cplus_hardregion_beta/online_model_final_aq0.238.pth +3 -0
- online/ga_8_baseline/online_baseline_stage1_ga8.yaml +63 -0
- online/ga_8_baseline/online_model_final_ga8_aq0.386.pth +3 -0
online/A_control/offline_baseline_stage1_ga32.yaml
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Offline tracking stage (DVIS_Plus_offline) at GA=32, 2/3 training budget.
|
| 2 |
+
#
|
| 3 |
+
# See configs/full/online/final_run/baseline_stage1_ga32.yaml for the full derivation.
|
| 4 |
+
# effective_batch = IMS_PER_BATCH * OPT_GA_STEPS = 8 * 32 = 256 (was 64 at GA=8)
|
| 5 |
+
# optimizer_steps = 106656 / 32 = 3333 (2/3 of 5000)
|
| 6 |
+
#
|
| 7 |
+
# MODEL.WEIGHTS chains from the GA=32 ONLINE run (not the GA=8 one), keeping the
|
| 8 |
+
# cascade internally consistent: stage-1 segmenter -> online_ga32 -> offline_ga32.
|
| 9 |
+
# This run therefore cannot start until full_online_ga32/model_final.pth exists.
|
| 10 |
+
#
|
| 11 |
+
# DATALOADER.NUM_WORKERS 16 -> 32: the GA=8 offline run spent ~61% of wall-clock
|
| 12 |
+
# blocked on the dataloader (median data_time 1.324s of a 2.155s iter), reading
|
| 13 |
+
# clips from the HDFS FUSE mount. PROJECTED speedup, not yet benchmarked.
|
| 14 |
+
_BASE_: baseline_stage1.yaml
|
| 15 |
+
|
| 16 |
+
MODEL:
|
| 17 |
+
WEIGHTS: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online_ga32/model_final.pth"
|
| 18 |
+
|
| 19 |
+
SOLVER:
|
| 20 |
+
MAX_ITER: 106656
|
| 21 |
+
STEPS: (74656, 96000)
|
| 22 |
+
WARMUP_ITERS: 12000
|
| 23 |
+
CHECKPOINT_PERIOD: 10000 # ~7h between saves; see the online ga32 config
|
| 24 |
+
|
| 25 |
+
DATALOADER:
|
| 26 |
+
NUM_WORKERS: 32
|
| 27 |
+
|
| 28 |
+
OPT_GA: True
|
| 29 |
+
OPT_GA_STEPS: 32
|
| 30 |
+
|
| 31 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_offline_ga32'
|
| 32 |
+
|
| 33 |
+
# In-training AQ / STQ / mIoU on the 120-clip val cache (dvis_Plus/aq_eval_hook.py).
|
| 34 |
+
# Period is in OPTIMIZER steps -> ~16 evals over the 3333-step run, ~60s each.
|
| 35 |
+
# NOTE: this is a FRONT center-crop subset, so its absolute AQ reads low (~0.15 vs
|
| 36 |
+
# the 0.386 in aq_500, which averages 5 cameras x 5 crops). Use it as a RELATIVE
|
| 37 |
+
# signal for whether association is improving.
|
| 38 |
+
OPT_AQ_EVAL_EVERY: 200
|
online/A_control/online_baseline_stage1_ga32.yaml
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Online tracking stage (DVIS_Plus_online) at GA=32, 2/3 training budget.
|
| 2 |
+
#
|
| 3 |
+
# MAX_ITER counts MICRO-iterations (one forward/backward per GAAMPTrainer.run_step,
|
| 4 |
+
# train_video.py:4763), NOT optimizer steps. Optimizer steps = MAX_ITER / OPT_GA_STEPS.
|
| 5 |
+
#
|
| 6 |
+
# effective_batch = IMS_PER_BATCH * OPT_GA_STEPS = 8 * 32 = 256 (was 64 at GA=8)
|
| 7 |
+
# optimizer_steps = 106656 / 32 = 3333 (2/3 of the 5000
|
| 8 |
+
# used by the GA=8 run)
|
| 9 |
+
#
|
| 10 |
+
# Schedule keys are all in micro-iterations, so they are set to hold the same
|
| 11 |
+
# optimizer-step fractions as the GA=8 baseline:
|
| 12 |
+
# STEPS (74656, 96000) -> opt steps 2333 / 3000 = 0.70 / 0.90 of the run
|
| 13 |
+
# WARMUP_ITERS 12000 -> 375 warmup opt steps, same ABSOLUTE count as the
|
| 14 |
+
# GA=8 baseline (warmup is about update count, so it
|
| 15 |
+
# is deliberately NOT scaled down with the budget)
|
| 16 |
+
# CHECKPOINT_PERIOD 10000 -> a save every 10000 micro-iters (= 312 optimizer
|
| 17 |
+
# steps, ~7h wall-clock), 10 checkpoints + final.
|
| 18 |
+
# NOT scaled to preserve the GA=8 checkpoint COUNT:
|
| 19 |
+
# this run is ~4x longer in wall-clock, so keeping
|
| 20 |
+
# the old count would mean ~14h between saves. This
|
| 21 |
+
# keeps the ~7h cadence of the GA=8 run instead.
|
| 22 |
+
#
|
| 23 |
+
# BASE_LR is UNCHANGED at 0.000283, following the existing GA sweep
|
| 24 |
+
# (final_propose_stage2_no_beta_ce_ga16/_ga32/_ga64 all held BASE_LR constant
|
| 25 |
+
# despite raising the effective batch).
|
| 26 |
+
#
|
| 27 |
+
# DATALOADER.NUM_WORKERS 16 -> 32. The GA=8 runs spent ~50% of wall-clock blocked
|
| 28 |
+
# on next(data_loader) (median data_time 1.287s of a 2.586s iter). Training data
|
| 29 |
+
# lives on the HDFS FUSE mount (dvis_Plus/data_video/datasets/waymo.py:2011), so
|
| 30 |
+
# workers block on network I/O rather than CPU; 8 GPUs x 32 = 256 worker procs on
|
| 31 |
+
# 192 cores is intentional oversubscription to hide that latency.
|
| 32 |
+
# NOTE: this speedup is PROJECTED, not yet benchmarked on this host.
|
| 33 |
+
_BASE_: baseline_stage1.yaml
|
| 34 |
+
|
| 35 |
+
SOLVER:
|
| 36 |
+
MAX_ITER: 106656
|
| 37 |
+
STEPS: (74656, 96000)
|
| 38 |
+
WARMUP_ITERS: 12000
|
| 39 |
+
CHECKPOINT_PERIOD: 10000
|
| 40 |
+
|
| 41 |
+
DATALOADER:
|
| 42 |
+
NUM_WORKERS: 32
|
| 43 |
+
|
| 44 |
+
OPT_GA: True
|
| 45 |
+
OPT_GA_STEPS: 32
|
| 46 |
+
|
| 47 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online_ga32'
|
| 48 |
+
|
| 49 |
+
# In-training AQ / STQ / mIoU on the 120-clip val cache (dvis_Plus/aq_eval_hook.py).
|
| 50 |
+
# Period is in OPTIMIZER steps -> ~16 evals over the 3333-step run, ~60s each.
|
| 51 |
+
# NOTE: this is a FRONT center-crop subset, so its absolute AQ reads low (~0.15 vs
|
| 52 |
+
# the 0.386 in aq_500, which averages 5 cameras x 5 crops). Use it as a RELATIVE
|
| 53 |
+
# signal for whether association is improving.
|
| 54 |
+
OPT_AQ_EVAL_EVERY: 200
|
online/B_proposeinit/offline_baseline_stage1_ga32_proposeinit.yaml
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# OFFLINE stage for ARM B (the init-swap arm).
|
| 2 |
+
#
|
| 3 |
+
# Cascade: stage-1 segmenter (propose_ga32) -> online arm B -> THIS.
|
| 4 |
+
#
|
| 5 |
+
# Identical to the offline control (baseline_stage1_ga32.yaml) in every respect --
|
| 6 |
+
# same GA, same 3333 optimizer steps, same LR, no teacher, no hard-region loss --
|
| 7 |
+
# except MODEL.WEIGHTS, which chains from arm B's online output instead of arm A's.
|
| 8 |
+
#
|
| 9 |
+
# offline control init : full_online_ga32/model_final.pth (from arm A)
|
| 10 |
+
# THIS init : full_online_ga32_proposeinit/model_final.pth (from arm B)
|
| 11 |
+
#
|
| 12 |
+
# So (offline B) - (offline control) asks whether the arm-B advantage -- if any --
|
| 13 |
+
# SURVIVES the refiner, or whether the refiner washes it out. That matters: the refiner
|
| 14 |
+
# is where almost all of the association gain lives (AQ 0.386 -> 0.503 at equal mIoU on
|
| 15 |
+
# the GA=8 runs), so a segmenter-level improvement is only interesting if it is still
|
| 16 |
+
# visible after refinement.
|
| 17 |
+
#
|
| 18 |
+
# BLOCKED until arm B produces full_online_ga32_proposeinit/model_final.pth.
|
| 19 |
+
_BASE_: baseline_stage1_ga32.yaml
|
| 20 |
+
|
| 21 |
+
MODEL:
|
| 22 |
+
WEIGHTS: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online_ga32_proposeinit/model_final.pth"
|
| 23 |
+
|
| 24 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_offline_ga32_proposeinit'
|
online/B_proposeinit/online_baseline_stage1_ga32_proposeinit.yaml
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ARM B: online tracker at GA=32, initialised from the PROPOSE (hard-region) segmenter.
|
| 2 |
+
#
|
| 3 |
+
# Identical to arm A (baseline_stage1_ga32.yaml) in EVERY respect -- same GA, same
|
| 4 |
+
# 3333 optimizer steps, same LR, same data, same schedule, no teacher, no hard-region
|
| 5 |
+
# loss in the online stage -- except ONE thing: MODEL.WEIGHTS.
|
| 6 |
+
#
|
| 7 |
+
# arm A init : baseline_stage1/model_final.pth
|
| 8 |
+
# the VANILLA stage-1 segmenter.
|
| 9 |
+
# arm B init : final_propose_stage2_no_beta_ce_ga32/model_final.pth
|
| 10 |
+
# the same segmenter further trained with the alpha/beta hard-region
|
| 11 |
+
# method (propose_ga32: OPT_TRAIN_LOSS=KD, DualMaskHeads decoder,
|
| 12 |
+
# plain XOR hard region, teacher = baseline_stage1).
|
| 13 |
+
#
|
| 14 |
+
# So B - A answers exactly one question:
|
| 15 |
+
# "Does a better (hard-region-trained) SEGMENTER produce a better TRACKER?"
|
| 16 |
+
# It does NOT add any hard-region loss to the online stage itself -- the tracker is
|
| 17 |
+
# trained with the plain loss_ce + loss_mask + loss_dice in both arms. The hard-region
|
| 18 |
+
# loss inside the online stage is a separate arm (baseline_stage1_ga32_hardregion.yaml).
|
| 19 |
+
#
|
| 20 |
+
# NOTE on the checkpoint: propose_ga32 was trained with the DualMaskHeads decoder, so
|
| 21 |
+
# its state_dict carries the extra beta params (boundary_decoder_norm.*, boundary_embed.*).
|
| 22 |
+
# The online stage uses the plain VideoMultiScaleMaskedTransformerDecoder_dvisPlus, so
|
| 23 |
+
# those keys are simply not consumed at load. The shared alpha weights -- decoder layers,
|
| 24 |
+
# decoder_norm, mask_embed, class_embed, pixel decoder, backbone -- load normally. That
|
| 25 |
+
# is intended: we want the segmenter that the hard-region training produced, not its
|
| 26 |
+
# train-only beta head (which is unused at inference anyway).
|
| 27 |
+
_BASE_: baseline_stage1_ga32.yaml
|
| 28 |
+
|
| 29 |
+
MODEL:
|
| 30 |
+
WEIGHTS: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/final_propose_stage2_no_beta_ce_ga32/model_final.pth"
|
| 31 |
+
|
| 32 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online_ga32_proposeinit'
|
online/C_hardregion/offline_baseline_stage1_ga32_hardregion.yaml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ARM B (OFFLINE): refiner at GA=32 WITH the teacher-guided hard-region loss.
|
| 2 |
+
#
|
| 3 |
+
# A/B against configs/full/offline/final_run/baseline_stage1_ga32.yaml. Inherits
|
| 4 |
+
# everything from it, so the two differ ONLY in the OPT_TRACKER_KD block.
|
| 5 |
+
#
|
| 6 |
+
# student init : full_online_ga32/model_final.pth (the arm-B-consistent cascade:
|
| 7 |
+
# stage1 -> online_ga32_hardregion -> offline_ga32_hardregion)
|
| 8 |
+
# TEACHER : full_offline/model_final.pth (the finished offline refiner,
|
| 9 |
+
# AQ 0.503 on aq_500) -- a full DVIS_Plus_offline, so its REFINER is
|
| 10 |
+
# run too and the hard region reflects REFINED masks.
|
| 11 |
+
# hard region : hard = GT XOR (teacher_mask > 0) -- plain XOR, as in propose_ga32.
|
| 12 |
+
#
|
| 13 |
+
# NOTE the offline layout: the criterion flattens the T frames into one tall image
|
| 14 |
+
# ('b q t h w -> b q () (t h) w'). GT, student and teacher are all flattened the same
|
| 15 |
+
# way, and the XOR is shape-agnostic, so the hard region is computed consistently.
|
| 16 |
+
#
|
| 17 |
+
# BLOCKED until the arm-B online run produces full_online_ga32/model_final.pth.
|
| 18 |
+
_BASE_: baseline_stage1_ga32.yaml
|
| 19 |
+
|
| 20 |
+
MODEL:
|
| 21 |
+
WEIGHTS: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online_ga32_hardregion/model_final.pth"
|
| 22 |
+
|
| 23 |
+
OPT_TRACKER_KD: True
|
| 24 |
+
OPT_TRACKER_BETA_HEAD: False
|
| 25 |
+
OPT_TRACKER_HR_CLASS_AWARE: False
|
| 26 |
+
|
| 27 |
+
OPT_ALPHA_MASK_WEIGHT: 0.1
|
| 28 |
+
OPT_ALPHA_DICE_WEIGHT: 1.0
|
| 29 |
+
OPT_BETA_MASK_WEIGHT: 1.0
|
| 30 |
+
OPT_BETA_DICE_WEIGHT: 3.0
|
| 31 |
+
OPT_HC_NUM_POINTS_DIVIDER: 5
|
| 32 |
+
|
| 33 |
+
OPT_TEACHER_CONFIG: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_offline/config.yaml"
|
| 34 |
+
OPT_TEACHER_WEIGHTS: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_offline/model_final.pth"
|
| 35 |
+
|
| 36 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_offline_ga32_hardregion'
|
online/C_hardregion/online_baseline_stage1_ga32_hardregion.yaml
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ARM C: online tracker at GA=32 with the teacher-guided hard-region loss (NO beta head).
|
| 2 |
+
#
|
| 3 |
+
# Arm C+ (baseline_stage1_ga32_hardregion_beta.yaml) inherits from this file and turns
|
| 4 |
+
# the beta head ON. C keeps it OFF, so C is architecturally IDENTICAL to arm A (zero
|
| 5 |
+
# extra params) and C - A isolates the LOSS alone; C+ - C then isolates the BETA HEAD.
|
| 6 |
+
#
|
| 7 |
+
# Inherits everything from arm A (baseline_stage1_ga32.yaml) -- same init, data,
|
| 8 |
+
# schedule, batch, LR, checkpointing -- and adds only the OPT_TRACKER_KD block below.
|
| 9 |
+
#
|
| 10 |
+
# student init : baseline_stage1/model_final.pth (SAME as arm A)
|
| 11 |
+
# TEACHER : full_online/model_final.pth (the finished GA=8 online tracker,
|
| 12 |
+
# AQ 0.386 on aq_500) -- a full DVIS_Plus_online, so its TRACKER is run
|
| 13 |
+
# too and the hard region reflects TRACKED masks, not segmenter masks.
|
| 14 |
+
# hard region : hard = GT XOR (teacher_mask > 0) -- plain, class-blind XOR, matching
|
| 15 |
+
# propose_stage2_no_beta_ce_ga32.yaml (which sets neither
|
| 16 |
+
# OPT_HARD_REGION_CLASS_AWARE nor OPT_HARD_REGION_SEMANTIC).
|
| 17 |
+
# losses : loss2_alpha_mask/dice on the tracker's existing mask head
|
| 18 |
+
# weights : copied verbatim from propose_ga32 (alpha 0.1/1.0; beta 1.0/3.0 are
|
| 19 |
+
# set here too but unused unless OPT_TRACKER_BETA_HEAD is on -- arm C+).
|
| 20 |
+
#
|
| 21 |
+
# C - A isolates the hard-region LOSS in the online stage.
|
| 22 |
+
# (B - A is the separate init-swap arm; see online_offline_exp.md.)
|
| 23 |
+
#
|
| 24 |
+
# BETA HEAD CAVEAT -- relevant to arm C+, which flips OPT_TRACKER_BETA_HEAD on.
|
| 25 |
+
# In the segmenter, alpha and beta are two QUERY STREAMS through shared decoder layers;
|
| 26 |
+
# they diverge because each builds its own PIXEL ATTENTION MASK from its own mask logits,
|
| 27 |
+
# which then steers its own query evolution. The tracker's queries cross-attend to FRAME
|
| 28 |
+
# EMBEDDINGS and every one of its attention calls passes memory_mask=None
|
| 29 |
+
# (tracker.py:315, 336, 373, 394, ...), so there is no such mask and a cloned second
|
| 30 |
+
# stream would be BIT-IDENTICAL to alpha. Beta here is therefore a second OUTPUT HEAD
|
| 31 |
+
# (boundary_decoder_norm + boundary_embed) on the shared final query state.
|
| 32 |
+
# What survives: beta still trains only on the hard region, and its gradients still flow
|
| 33 |
+
# back through the shared transformer stack -- the mechanism by which beta shapes the
|
| 34 |
+
# trunk in the segmenter. What is lost: beta cannot specialise its own queries.
|
| 35 |
+
# Expect a weaker effect than the segmenter's beta.
|
| 36 |
+
# See dvis_Plus/tracker_hard_region.py.
|
| 37 |
+
_BASE_: baseline_stage1_ga32.yaml
|
| 38 |
+
|
| 39 |
+
OPT_TRACKER_KD: True
|
| 40 |
+
OPT_TRACKER_BETA_HEAD: False # arm C+ overrides this to True
|
| 41 |
+
OPT_TRACKER_HR_CLASS_AWARE: False # plain XOR, as in propose_ga32
|
| 42 |
+
|
| 43 |
+
OPT_ALPHA_MASK_WEIGHT: 0.1
|
| 44 |
+
OPT_ALPHA_DICE_WEIGHT: 1.0
|
| 45 |
+
OPT_BETA_MASK_WEIGHT: 1.0
|
| 46 |
+
OPT_BETA_DICE_WEIGHT: 3.0
|
| 47 |
+
OPT_HC_NUM_POINTS_DIVIDER: 5 # hard-region point budget = TRAIN_NUM_POINTS / 5
|
| 48 |
+
|
| 49 |
+
OPT_TEACHER_CONFIG: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online/config.yaml"
|
| 50 |
+
OPT_TEACHER_WEIGHTS: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online/model_final.pth"
|
| 51 |
+
|
| 52 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online_ga32_hardregion'
|
online/Cplus_hardregion_beta/offline_baseline_stage1_ga32_hardregion_beta.yaml
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# OFFLINE stage for ARM C+ (hard-region loss + beta head).
|
| 2 |
+
#
|
| 3 |
+
# Cascade: stage-1 segmenter -> online arm C+ -> THIS.
|
| 4 |
+
#
|
| 5 |
+
# Identical to the offline hard-region arm (baseline_stage1_ga32_hardregion.yaml)
|
| 6 |
+
# except it turns the beta head on, mirroring what arm C+ does in the online stage --
|
| 7 |
+
# so each offline run carries the SAME treatment as the online arm it chains from.
|
| 8 |
+
#
|
| 9 |
+
# init : full_online_ga32_hardregion_beta/model_final.pth (from arm C+)
|
| 10 |
+
# TEACHER : full_offline/model_final.pth -- the finished GA=8 offline refiner
|
| 11 |
+
# (AQ 0.503 on aq_500). A full DVIS_Plus_offline, so its REFINER is run too
|
| 12 |
+
# and the hard region reflects REFINED masks, not tracker masks.
|
| 13 |
+
# losses : loss2_alpha_mask/dice on the refiner's existing mask head
|
| 14 |
+
# loss2_beta_mask/dice on the refiner's beta head (hard region only)
|
| 15 |
+
#
|
| 16 |
+
# The beta head on the refiner has the same limitation as on the tracker: it is a second
|
| 17 |
+
# OUTPUT head on the shared query state, not a second query stream, because there is no
|
| 18 |
+
# pixel attention mask for two streams to diverge on. See dvis_Plus/tracker_hard_region.py.
|
| 19 |
+
#
|
| 20 |
+
# Checkpoint note: arm C+'s online model_final carries tracker beta params
|
| 21 |
+
# (tracker.boundary_*). The offline stage loads the tracker and freezes it; with
|
| 22 |
+
# OPT_TRACKER_BETA_HEAD on, those keys are consumed. Beta is train-only in any case --
|
| 23 |
+
# inference reads the alpha head -- so it does not affect the offline tracker's masks.
|
| 24 |
+
#
|
| 25 |
+
# BLOCKED until arm C+ produces full_online_ga32_hardregion_beta/model_final.pth.
|
| 26 |
+
_BASE_: baseline_stage1_ga32_hardregion.yaml
|
| 27 |
+
|
| 28 |
+
MODEL:
|
| 29 |
+
WEIGHTS: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online_ga32_hardregion_beta/model_final.pth"
|
| 30 |
+
|
| 31 |
+
OPT_TRACKER_BETA_HEAD: True
|
| 32 |
+
|
| 33 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_offline_ga32_hardregion_beta'
|
online/Cplus_hardregion_beta/online_baseline_stage1_ga32_hardregion_beta.yaml
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ARM C: online tracker at GA=32, hard-region loss + the BETA HEAD.
|
| 2 |
+
#
|
| 3 |
+
# Identical to arm B (baseline_stage1_ga32_hardregion.yaml) except it turns the beta
|
| 4 |
+
# head on, so the three online arms isolate two variables:
|
| 5 |
+
#
|
| 6 |
+
# arm A baseline_stage1_ga32.yaml control: no teacher, no hard region
|
| 7 |
+
# arm B ..._ga32_hardregion.yaml + hard-region loss (loss2_alpha_*)
|
| 8 |
+
# arm C ..._ga32_hardregion_beta.yaml + hard-region loss + beta head
|
| 9 |
+
# (loss2_alpha_* AND loss2_beta_*)
|
| 10 |
+
#
|
| 11 |
+
# B - A isolates the LOSS (A and B are architecturally identical).
|
| 12 |
+
# C - B isolates the BETA HEAD (same loss, extra head + extra params).
|
| 13 |
+
#
|
| 14 |
+
# On the beta head here: it is a second OUTPUT head (boundary_decoder_norm +
|
| 15 |
+
# boundary_embed) reading the same final query state, not a second query stream.
|
| 16 |
+
# The segmenter's beta gets its own query stream because each stream builds its own
|
| 17 |
+
# pixel attention mask from its own mask logits; the tracker's queries cross-attend to
|
| 18 |
+
# frame embeddings with memory_mask=None, so there is no such mask and a cloned stream
|
| 19 |
+
# would be bit-identical to alpha. Beta still trains only on the hard region and its
|
| 20 |
+
# gradients still flow back through the shared transformer stack -- the mechanism that
|
| 21 |
+
# shapes the trunk in the segmenter -- it just cannot specialise its own queries.
|
| 22 |
+
# Expect a smaller effect than the segmenter's beta. See dvis_Plus/tracker_hard_region.py.
|
| 23 |
+
#
|
| 24 |
+
# Beta weights are propose_ga32's (1.0 mask / 3.0 dice).
|
| 25 |
+
_BASE_: baseline_stage1_ga32_hardregion.yaml
|
| 26 |
+
|
| 27 |
+
OPT_TRACKER_BETA_HEAD: True
|
| 28 |
+
|
| 29 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online_ga32_hardregion_beta'
|
online/Cplus_hardregion_beta/online_model_0009999_early_aq0.16.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:50ad504a647d490e15c73b8a06729e91d19b4b3606d542048e46675381f79c00
|
| 3 |
+
size 3223722042
|
online/Cplus_hardregion_beta/online_model_final_aq0.238.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e69a3464618a62772025af13edb9006665dfc62579e7fa0d1a75217a105d7a56
|
| 3 |
+
size 3223722042
|
online/ga_8_baseline/online_baseline_stage1_ga8.yaml
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Online tracking stage (DVIS_Plus_online), machine-adapted for this host.
|
| 2 |
+
# Mirrors configs/full/mask2former/final_run/baseline_stage1.yaml (the segmenter
|
| 3 |
+
# stage-1 config) but trains the online tracker on top of the trained segmenter:
|
| 4 |
+
# - MODEL.WEIGHTS -> the trained stage-1 segmenter (backbone + sem_seg_head are
|
| 5 |
+
# loaded and frozen; the tracker module is trained).
|
| 6 |
+
# - same dataset / data pipeline as the segmenter (waymo_train_all_5_views).
|
| 7 |
+
# - OUTPUT_DIR -> .../new_waymo_config1/full_online
|
| 8 |
+
# META_ARCHITECTURE: DVIS_Plus_online comes from the _BASE_ below.
|
| 9 |
+
_BASE_: ../DVIS_Plus_Online_R50.yaml
|
| 10 |
+
MODEL:
|
| 11 |
+
WEIGHTS: "/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/baseline_stage1/model_final.pth"
|
| 12 |
+
BACKBONE:
|
| 13 |
+
NAME: "D2VitAdapterDinoV2"
|
| 14 |
+
VIT_ADAPTER:
|
| 15 |
+
FINETUNE: False
|
| 16 |
+
NAME: 'vitl'
|
| 17 |
+
VIT_WEIGHT: '/mnt/hdfs/andrew.estornell/multi_modal/repo_backup/weights/work_dirs/dinov2_vitl14_pretrain_.pth'
|
| 18 |
+
FREEZE_VIT: True
|
| 19 |
+
PIXEL_MEAN: [123.675, 116.280, 103.530]
|
| 20 |
+
PIXEL_STD: [58.395, 57.120, 57.375]
|
| 21 |
+
MASK_FORMER:
|
| 22 |
+
NUM_OBJECT_QUERIES: 200
|
| 23 |
+
NO_OBJECT_WEIGHT: 0.1
|
| 24 |
+
TRANSFORMER_DECODER_NAME: "VideoMultiScaleMaskedTransformerDecoder_dvisPlus"
|
| 25 |
+
CL_PLUGIN:
|
| 26 |
+
NUM_NEGATIVES: 199
|
| 27 |
+
SEM_SEG_HEAD:
|
| 28 |
+
NUM_CLASSES: 29
|
| 29 |
+
SOLVER:
|
| 30 |
+
IMS_PER_BATCH: 8
|
| 31 |
+
OPTIMIZER: "ADAMW"
|
| 32 |
+
BASE_LR: 0.000283
|
| 33 |
+
MAX_ITER: 40000
|
| 34 |
+
STEPS: (28000, 36000)
|
| 35 |
+
GAMMA: 0.1
|
| 36 |
+
CHECKPOINT_PERIOD: 8000
|
| 37 |
+
WEIGHT_DECAY: 0.05
|
| 38 |
+
WARMUP_FACTOR: 0.001
|
| 39 |
+
WARMUP_ITERS: 3000
|
| 40 |
+
|
| 41 |
+
OPT_FLAG_ABSENT_IDS: True
|
| 42 |
+
|
| 43 |
+
DATALOADER:
|
| 44 |
+
FILTER_EMPTY_ANNOTATIONS: False
|
| 45 |
+
NUM_WORKERS: 16
|
| 46 |
+
INPUT:
|
| 47 |
+
MIN_SIZE_TRAIN: (960, 1120, 1280)
|
| 48 |
+
MAX_SIZE_TRAIN: 1920
|
| 49 |
+
SAMPLING_FRAME_NUM: 5
|
| 50 |
+
SAMPLING_FRAME_RANGE: 5
|
| 51 |
+
MIN_SIZE_TEST: 1280
|
| 52 |
+
OPT_IF_ADD_RANDOM_CROP: True
|
| 53 |
+
OPT_IF_ADD_RANDOM_FLIP: True
|
| 54 |
+
|
| 55 |
+
OPT_GA: True
|
| 56 |
+
OPT_GA_STEPS: 8
|
| 57 |
+
|
| 58 |
+
OPT_DATASET: "waymo_train_all_5_views"
|
| 59 |
+
OPT_VAL_DATASET: "waymo_train_5_views"
|
| 60 |
+
OPT_ADD_DAUG: "add_data_aug"
|
| 61 |
+
OPT_ATTN_MASK_METHOD: "attn_mask"
|
| 62 |
+
|
| 63 |
+
OUTPUT_DIR: '/mnt/hdfs/andrew.estornell/multi_modal/DVIS_Plus/new_output_dir/new_waymo_config1/full_online'
|
online/ga_8_baseline/online_model_final_ga8_aq0.386.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e4e80cafc185d62ca7db4121aeb2809450312a661c0982771a7653b1a9d03661
|
| 3 |
+
size 1715022485
|