Michael Rabinovich Cursor commited on
Commit ·
987f284
1
Parent(s): ec67f32
Submit gate: accept unparseable candidates (eval scores them invalid/0)
Browse filesA present-but-unloadable output.step (empty, or parses to no geometry) was
rejecting the entire submission. The evaluator's CAD-validity gate already
scores any non-valid solid as cad_score=0 (status=invalid), so a bad candidate
is a legitimate zero-scoring outcome, not a submission error. The gate now logs
and lets it through instead of blocking the whole zip.
Co-authored-by: Cursor <cursoragent@cursor.com>
submit.py
CHANGED
|
@@ -716,22 +716,32 @@ def _validate_candidates_parseable(unpacked: Path, fixture_names: set[str]) -> N
|
|
| 716 |
# Missing output is a valid benchmark outcome: the evaluator writes
|
| 717 |
# status="missing" and the fixture contributes cad_score=0.
|
| 718 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 719 |
if candidate.stat().st_size == 0:
|
| 720 |
-
|
| 721 |
-
|
|
|
|
| 722 |
)
|
|
|
|
| 723 |
suffix = candidate.suffix.lower()
|
| 724 |
if suffix in {".step", ".stp"}:
|
| 725 |
try:
|
| 726 |
parse_step(candidate)
|
| 727 |
-
except
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
|
|
|
| 732 |
else: # pragma: no cover - _candidate_path constrains suffixes
|
| 733 |
-
|
| 734 |
-
|
|
|
|
| 735 |
)
|
| 736 |
|
| 737 |
with ThreadPoolExecutor(
|
|
|
|
| 716 |
# Missing output is a valid benchmark outcome: the evaluator writes
|
| 717 |
# status="missing" and the fixture contributes cad_score=0.
|
| 718 |
return
|
| 719 |
+
# A present-but-unloadable candidate is NOT a submission error. The
|
| 720 |
+
# evaluator's CAD-validity gate scores any non-valid solid as
|
| 721 |
+
# cad_score=0 (status="invalid"), so an empty or unparseable file is a
|
| 722 |
+
# legitimate (zero-scoring) outcome, not a reason to reject the whole
|
| 723 |
+
# submission. We only log here; the gate never blocks on candidate
|
| 724 |
+
# content, so one bad part can't sink an otherwise-valid submission.
|
| 725 |
if candidate.stat().st_size == 0:
|
| 726 |
+
logger.warning(
|
| 727 |
+
"Sample %s has an empty %s; will score 0 (invalid).",
|
| 728 |
+
name, candidate.name,
|
| 729 |
)
|
| 730 |
+
return
|
| 731 |
suffix = candidate.suffix.lower()
|
| 732 |
if suffix in {".step", ".stp"}:
|
| 733 |
try:
|
| 734 |
parse_step(candidate)
|
| 735 |
+
except Exception as e: # noqa: BLE001 - non-fatal; eval scores it 0
|
| 736 |
+
logger.warning(
|
| 737 |
+
"Sample %s has an %s that is not loadable as STEP (%s); "
|
| 738 |
+
"will score 0 (invalid).",
|
| 739 |
+
name, candidate.name, e,
|
| 740 |
+
)
|
| 741 |
else: # pragma: no cover - _candidate_path constrains suffixes
|
| 742 |
+
logger.warning(
|
| 743 |
+
"Sample %s uses unexpected candidate file %s; will score 0.",
|
| 744 |
+
name, candidate.name,
|
| 745 |
)
|
| 746 |
|
| 747 |
with ThreadPoolExecutor(
|