xsponenta Claude Opus 4.7 commited on
Commit ·
bd280f7
1
Parent(s): b6bc99a
Revert TTA: 3x inference timed out on HF Space (b6bc99a -> all zeros)
Browse filesSet USE_TTA=False. The TTA strict variant locally showed +0.003 mean and
+0.029 q5, but the 3x inference cost exceeded HF Space's evaluation time
limit, returning 0.0 for every sample. Pipeline now runs the single-seed
fast path it used in 2df06c6 (the proven 0.4815 leaderboard config).
Also leaves USE_ENSEMBLE=False with infrastructure preserved: a fine-tune
attempt with heavy augmentation (jitter 0.01 + drop 0.1) produced a model
0.040 hss_mean WORSE than the original on local 50-sample A/B, so the
ensemble code in script.py + ensemble.py + tta.py is wired but disabled
pending a better-trained 2nd checkpoint.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
script.py
CHANGED
|
@@ -59,14 +59,25 @@ CONF_THRESH = 0.4
|
|
| 59 |
MERGE_THRESH = 0.4
|
| 60 |
SNAP_RADIUS = 0.5
|
| 61 |
|
| 62 |
-
# Test-time augmentation: 3 priority-sample seeds + Hungarian matching
|
| 63 |
-
#
|
| 64 |
-
#
|
| 65 |
-
#
|
| 66 |
-
|
|
|
|
| 67 |
TTA_SEEDS = (2718, 31415, 42)
|
| 68 |
TTA_MIN_PASSES = 2
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
|
| 71 |
def fuse_and_sample(sample, cfg, rng):
|
| 72 |
"""Run point fusion + priority sampling on a raw dataset sample.
|
|
@@ -380,6 +391,17 @@ if __name__ == "__main__":
|
|
| 380 |
model = load_model(checkpoint_path, device)
|
| 381 |
print(f"Model loaded: {sum(p.numel() for p in model.parameters()):,} params")
|
| 382 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 383 |
# Point fusion config
|
| 384 |
cfg = FuserConfig()
|
| 385 |
rng = np.random.RandomState(2718)
|
|
@@ -410,7 +432,44 @@ if __name__ == "__main__":
|
|
| 410 |
pred_status = "ok"
|
| 411 |
n_fused_pts = 0
|
| 412 |
|
| 413 |
-
if USE_TTA:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 414 |
# Multi-seed TTA: fuse + predict 3 times, Hungarian-match segments
|
| 415 |
# across passes, drop those without min_passes agreement.
|
| 416 |
try:
|
|
|
|
| 59 |
MERGE_THRESH = 0.4
|
| 60 |
SNAP_RADIUS = 0.5
|
| 61 |
|
| 62 |
+
# Test-time augmentation: 3 priority-sample seeds + Hungarian matching.
|
| 63 |
+
# Local 100-sample A/B was +0.003 mean / +0.029 q5 vs single-pass, BUT the
|
| 64 |
+
# 3x inference cost (commit b6bc99a) timed out on HF Space and produced
|
| 65 |
+
# all-zero scores. Disabled. Re-enable only with single-seed (2x not 3x)
|
| 66 |
+
# or after profiling shows the HF Space can fit 3x within its time limit.
|
| 67 |
+
USE_TTA = False
|
| 68 |
TTA_SEEDS = (2718, 31415, 42)
|
| 69 |
TTA_MIN_PASSES = 2
|
| 70 |
|
| 71 |
+
# Multi-checkpoint ensemble: tried 2026-05-23 with checkpoint2.pt fine-tuned
|
| 72 |
+
# under heavy aug (jitter 0.01 + drop 0.1) for 20k steps from checkpoint.pt.
|
| 73 |
+
# Local 50-sample A/B (new model alone vs original alone): mean dropped by
|
| 74 |
+
# 0.040 with 20 big losses vs 5 big wins -- the aug degraded model accuracy.
|
| 75 |
+
# Ensemble of [original + new] therefore can't improve over original alone.
|
| 76 |
+
# Keep flag disabled; need a more careful training run before re-enabling.
|
| 77 |
+
USE_ENSEMBLE = False
|
| 78 |
+
ENSEMBLE_MIN_PASSES = 1
|
| 79 |
+
TTA_PLUS_ENSEMBLE_MIN_PASSES = 2
|
| 80 |
+
|
| 81 |
|
| 82 |
def fuse_and_sample(sample, cfg, rng):
|
| 83 |
"""Run point fusion + priority sampling on a raw dataset sample.
|
|
|
|
| 391 |
model = load_model(checkpoint_path, device)
|
| 392 |
print(f"Model loaded: {sum(p.numel() for p in model.parameters()):,} params")
|
| 393 |
|
| 394 |
+
# Optional: load 2nd checkpoint for ensemble inference
|
| 395 |
+
ensemble_models = None
|
| 396 |
+
if USE_ENSEMBLE:
|
| 397 |
+
checkpoint2_path = SCRIPT_DIR / "checkpoint2.pt"
|
| 398 |
+
if checkpoint2_path.exists() and checkpoint2_path.stat().st_size > 1000:
|
| 399 |
+
model2 = load_model(checkpoint2_path, device)
|
| 400 |
+
ensemble_models = [model, model2]
|
| 401 |
+
print(f"Ensemble: loaded 2 models for cross-checkpoint averaging")
|
| 402 |
+
else:
|
| 403 |
+
print(f"USE_ENSEMBLE=True but checkpoint2.pt not present; running single-model")
|
| 404 |
+
|
| 405 |
# Point fusion config
|
| 406 |
cfg = FuserConfig()
|
| 407 |
rng = np.random.RandomState(2718)
|
|
|
|
| 432 |
pred_status = "ok"
|
| 433 |
n_fused_pts = 0
|
| 434 |
|
| 435 |
+
if ensemble_models is not None and USE_TTA:
|
| 436 |
+
# 2-model ensemble × multi-seed TTA: 2 * len(seeds) total passes.
|
| 437 |
+
# Strict cross-pass agreement filters spurious segments.
|
| 438 |
+
try:
|
| 439 |
+
from ensemble import predict_sample_ensemble
|
| 440 |
+
pred_v, pred_e = predict_sample_ensemble(
|
| 441 |
+
sample, cfg, ensemble_models, device,
|
| 442 |
+
seeds=TTA_SEEDS,
|
| 443 |
+
min_passes_for_keep=TTA_PLUS_ENSEMBLE_MIN_PASSES,
|
| 444 |
+
)
|
| 445 |
+
if torch.cuda.is_available():
|
| 446 |
+
torch.cuda.empty_cache()
|
| 447 |
+
except Exception as e:
|
| 448 |
+
import traceback
|
| 449 |
+
print(f" Ensemble+TTA failed for {order_id}:\n{traceback.format_exc()}")
|
| 450 |
+
pred_v, pred_e = empty_solution()
|
| 451 |
+
pred_status = "ensemble_tta_failed"
|
| 452 |
+
if torch.cuda.is_available():
|
| 453 |
+
torch.cuda.empty_cache()
|
| 454 |
+
elif ensemble_models is not None:
|
| 455 |
+
# 2-model ensemble, single seed
|
| 456 |
+
try:
|
| 457 |
+
from ensemble import predict_sample_ensemble
|
| 458 |
+
pred_v, pred_e = predict_sample_ensemble(
|
| 459 |
+
sample, cfg, ensemble_models, device,
|
| 460 |
+
seeds=(2718,),
|
| 461 |
+
min_passes_for_keep=ENSEMBLE_MIN_PASSES,
|
| 462 |
+
)
|
| 463 |
+
if torch.cuda.is_available():
|
| 464 |
+
torch.cuda.empty_cache()
|
| 465 |
+
except Exception as e:
|
| 466 |
+
import traceback
|
| 467 |
+
print(f" Ensemble failed for {order_id}:\n{traceback.format_exc()}")
|
| 468 |
+
pred_v, pred_e = empty_solution()
|
| 469 |
+
pred_status = "ensemble_failed"
|
| 470 |
+
if torch.cuda.is_available():
|
| 471 |
+
torch.cuda.empty_cache()
|
| 472 |
+
elif USE_TTA:
|
| 473 |
# Multi-seed TTA: fuse + predict 3 times, Hungarian-match segments
|
| 474 |
# across passes, drop those without min_passes agreement.
|
| 475 |
try:
|