xsponenta Claude Opus 4.7 commited on
Commit
bd280f7
·
1 Parent(s): b6bc99a

Revert TTA: 3x inference timed out on HF Space (b6bc99a -> all zeros)

Browse files

Set USE_TTA=False. The TTA strict variant locally showed +0.003 mean and
+0.029 q5, but the 3x inference cost exceeded HF Space's evaluation time
limit, returning 0.0 for every sample. Pipeline now runs the single-seed
fast path it used in 2df06c6 (the proven 0.4815 leaderboard config).

Also leaves USE_ENSEMBLE=False with infrastructure preserved: a fine-tune
attempt with heavy augmentation (jitter 0.01 + drop 0.1) produced a model
0.040 hss_mean WORSE than the original on local 50-sample A/B, so the
ensemble code in script.py + ensemble.py + tta.py is wired but disabled
pending a better-trained 2nd checkpoint.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Files changed (1) hide show
  1. script.py +65 -6
script.py CHANGED
@@ -59,14 +59,25 @@ CONF_THRESH = 0.4
59
  MERGE_THRESH = 0.4
60
  SNAP_RADIUS = 0.5
61
 
62
- # Test-time augmentation: 3 priority-sample seeds + Hungarian matching with
63
- # strict 3-pass agreement (min_passes_for_keep=2). Local 100-sample A/B:
64
- # q5 0.062 -> 0.091 (+47%), mean +0.003. Costs 3x inference time but
65
- # strict filter dramatically improves precision on hard scenes.
66
- USE_TTA = True
 
67
  TTA_SEEDS = (2718, 31415, 42)
68
  TTA_MIN_PASSES = 2
69
 
 
 
 
 
 
 
 
 
 
 
70
 
71
  def fuse_and_sample(sample, cfg, rng):
72
  """Run point fusion + priority sampling on a raw dataset sample.
@@ -380,6 +391,17 @@ if __name__ == "__main__":
380
  model = load_model(checkpoint_path, device)
381
  print(f"Model loaded: {sum(p.numel() for p in model.parameters()):,} params")
382
 
 
 
 
 
 
 
 
 
 
 
 
383
  # Point fusion config
384
  cfg = FuserConfig()
385
  rng = np.random.RandomState(2718)
@@ -410,7 +432,44 @@ if __name__ == "__main__":
410
  pred_status = "ok"
411
  n_fused_pts = 0
412
 
413
- if USE_TTA:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
414
  # Multi-seed TTA: fuse + predict 3 times, Hungarian-match segments
415
  # across passes, drop those without min_passes agreement.
416
  try:
 
59
  MERGE_THRESH = 0.4
60
  SNAP_RADIUS = 0.5
61
 
62
+ # Test-time augmentation: 3 priority-sample seeds + Hungarian matching.
63
+ # Local 100-sample A/B was +0.003 mean / +0.029 q5 vs single-pass, BUT the
64
+ # 3x inference cost (commit b6bc99a) timed out on HF Space and produced
65
+ # all-zero scores. Disabled. Re-enable only with single-seed (2x not 3x)
66
+ # or after profiling shows the HF Space can fit 3x within its time limit.
67
+ USE_TTA = False
68
  TTA_SEEDS = (2718, 31415, 42)
69
  TTA_MIN_PASSES = 2
70
 
71
+ # Multi-checkpoint ensemble: tried 2026-05-23 with checkpoint2.pt fine-tuned
72
+ # under heavy aug (jitter 0.01 + drop 0.1) for 20k steps from checkpoint.pt.
73
+ # Local 50-sample A/B (new model alone vs original alone): mean dropped by
74
+ # 0.040 with 20 big losses vs 5 big wins -- the aug degraded model accuracy.
75
+ # Ensemble of [original + new] therefore can't improve over original alone.
76
+ # Keep flag disabled; need a more careful training run before re-enabling.
77
+ USE_ENSEMBLE = False
78
+ ENSEMBLE_MIN_PASSES = 1
79
+ TTA_PLUS_ENSEMBLE_MIN_PASSES = 2
80
+
81
 
82
  def fuse_and_sample(sample, cfg, rng):
83
  """Run point fusion + priority sampling on a raw dataset sample.
 
391
  model = load_model(checkpoint_path, device)
392
  print(f"Model loaded: {sum(p.numel() for p in model.parameters()):,} params")
393
 
394
+ # Optional: load 2nd checkpoint for ensemble inference
395
+ ensemble_models = None
396
+ if USE_ENSEMBLE:
397
+ checkpoint2_path = SCRIPT_DIR / "checkpoint2.pt"
398
+ if checkpoint2_path.exists() and checkpoint2_path.stat().st_size > 1000:
399
+ model2 = load_model(checkpoint2_path, device)
400
+ ensemble_models = [model, model2]
401
+ print(f"Ensemble: loaded 2 models for cross-checkpoint averaging")
402
+ else:
403
+ print(f"USE_ENSEMBLE=True but checkpoint2.pt not present; running single-model")
404
+
405
  # Point fusion config
406
  cfg = FuserConfig()
407
  rng = np.random.RandomState(2718)
 
432
  pred_status = "ok"
433
  n_fused_pts = 0
434
 
435
+ if ensemble_models is not None and USE_TTA:
436
+ # 2-model ensemble × multi-seed TTA: 2 * len(seeds) total passes.
437
+ # Strict cross-pass agreement filters spurious segments.
438
+ try:
439
+ from ensemble import predict_sample_ensemble
440
+ pred_v, pred_e = predict_sample_ensemble(
441
+ sample, cfg, ensemble_models, device,
442
+ seeds=TTA_SEEDS,
443
+ min_passes_for_keep=TTA_PLUS_ENSEMBLE_MIN_PASSES,
444
+ )
445
+ if torch.cuda.is_available():
446
+ torch.cuda.empty_cache()
447
+ except Exception as e:
448
+ import traceback
449
+ print(f" Ensemble+TTA failed for {order_id}:\n{traceback.format_exc()}")
450
+ pred_v, pred_e = empty_solution()
451
+ pred_status = "ensemble_tta_failed"
452
+ if torch.cuda.is_available():
453
+ torch.cuda.empty_cache()
454
+ elif ensemble_models is not None:
455
+ # 2-model ensemble, single seed
456
+ try:
457
+ from ensemble import predict_sample_ensemble
458
+ pred_v, pred_e = predict_sample_ensemble(
459
+ sample, cfg, ensemble_models, device,
460
+ seeds=(2718,),
461
+ min_passes_for_keep=ENSEMBLE_MIN_PASSES,
462
+ )
463
+ if torch.cuda.is_available():
464
+ torch.cuda.empty_cache()
465
+ except Exception as e:
466
+ import traceback
467
+ print(f" Ensemble failed for {order_id}:\n{traceback.format_exc()}")
468
+ pred_v, pred_e = empty_solution()
469
+ pred_status = "ensemble_failed"
470
+ if torch.cuda.is_available():
471
+ torch.cuda.empty_cache()
472
+ elif USE_TTA:
473
  # Multi-seed TTA: fuse + predict 3 times, Hungarian-match segments
474
  # across passes, drop those without min_passes agreement.
475
  try: