submit: default validation + hf_username fields on the pending row
Browse filesBundle 1+2 C4. `_build_pending_row` adds three new keys matching the
schema bump from C1, in the schema.md column order:
- validation_status = "unvalidated" (every new row starts in the
unvalidated tier; maintainers promote via the dataset repo after
methodology review).
- validation_method = None (set by the maintainer at promotion time
to one of code | traces | api | manual).
- hf_username = None (placeholder; C10 plumbs through the value
from gradio.OAuthProfile.username once the LoginButton lands).
Pre-existing fields, submitted_at stamping, the cadgenbench version
+ data revision tags, the score-shaped null fields, the blob URL,
and the sha256 are untouched. Dedup gate still pivots on
submission_sha256 exactly as before.
tests/test_submit.py covers the C4 acceptance:
- new-row defaults: validation_status is "unvalidated",
validation_method + hf_username are None;
- sha256 path passthrough: the hash argument lands on the row's
submission_sha256 key intact (existing dedup gate's row-level
half is unchanged);
- existing-metadata regression guard: submission_id, status,
submitter_name, submission_name, agent_url, notes,
submission_blob_url, cadgenbench_data_revision, and every
score-shaped null field all keep their pre-C4 values.
`_resolve_data_revision` is monkeypatched so the suite has zero
Hub I/O; cadgenbench.__version__ + datetime.now both stay live
(pure / process-local). 7/7 unit tests green locally (3 new + 4
existing leaderboard).
- submit.py +9 -0
- tests/test_submit.py +88 -0
|
@@ -401,6 +401,12 @@ def _build_pending_row(
|
|
| 401 |
metadata + ``status: pending`` + ``submission_blob_url`` +
|
| 402 |
``submission_sha256``; every score-shaped field is ``null`` until
|
| 403 |
the worker flips the row.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 404 |
"""
|
| 405 |
return {
|
| 406 |
"submission_id": submission_id,
|
|
@@ -421,6 +427,9 @@ def _build_pending_row(
|
|
| 421 |
"per_fixture_breakdown": None,
|
| 422 |
"submission_blob_url": blob_url,
|
| 423 |
"submission_sha256": submission_sha256,
|
|
|
|
|
|
|
|
|
|
| 424 |
}
|
| 425 |
|
| 426 |
|
|
|
|
| 401 |
metadata + ``status: pending`` + ``submission_blob_url`` +
|
| 402 |
``submission_sha256``; every score-shaped field is ``null`` until
|
| 403 |
the worker flips the row.
|
| 404 |
+
|
| 405 |
+
Validation-tier fields default per the validation-policy decision
|
| 406 |
+
doc: ``validation_status: "unvalidated"`` (maintainers promote
|
| 407 |
+
post-eval), ``validation_method: None``. ``hf_username`` is also
|
| 408 |
+
``None`` here; C10 wires it through from ``gr.OAuthProfile`` once
|
| 409 |
+
the LoginButton lands.
|
| 410 |
"""
|
| 411 |
return {
|
| 412 |
"submission_id": submission_id,
|
|
|
|
| 427 |
"per_fixture_breakdown": None,
|
| 428 |
"submission_blob_url": blob_url,
|
| 429 |
"submission_sha256": submission_sha256,
|
| 430 |
+
"validation_status": "unvalidated",
|
| 431 |
+
"validation_method": None,
|
| 432 |
+
"hf_username": None,
|
| 433 |
}
|
| 434 |
|
| 435 |
|
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Unit tests for the submit-tab pending-row builder.
|
| 2 |
+
|
| 3 |
+
C4 contract: ``_build_pending_row`` defaults the three Bundle 1+2
|
| 4 |
+
schema fields (``validation_status="unvalidated"``,
|
| 5 |
+
``validation_method=None``, ``hf_username=None``) and keeps the
|
| 6 |
+
existing metadata + sha256 fields intact.
|
| 7 |
+
|
| 8 |
+
Hub I/O (``_resolve_data_revision`` reads from
|
| 9 |
+
``HfApi.dataset_info``) is monkeypatched out, so the suite has zero
|
| 10 |
+
network traffic.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import submit
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _stub_meta() -> dict:
|
| 18 |
+
"""Minimum meta.json shape that survives ``_load_and_validate_meta``."""
|
| 19 |
+
return {
|
| 20 |
+
"submitter_name": "team-test",
|
| 21 |
+
"submission_name": "Stub Agent v1",
|
| 22 |
+
"agent_url": "https://github.com/example/stub-agent",
|
| 23 |
+
"notes": "test row, not a real submission",
|
| 24 |
+
"agree_to_publish": True,
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def test_pending_row_defaults_new_fields(monkeypatch):
|
| 29 |
+
"""Three Bundle 1+2 fields land with their schema defaults."""
|
| 30 |
+
monkeypatch.setattr(submit, "_resolve_data_revision", lambda: "test-rev")
|
| 31 |
+
row = submit._build_pending_row(
|
| 32 |
+
submission_id="sub-test-x",
|
| 33 |
+
meta=_stub_meta(),
|
| 34 |
+
blob_url="https://huggingface.co/datasets/example/sub-test-x.zip",
|
| 35 |
+
submission_sha256="a" * 64,
|
| 36 |
+
)
|
| 37 |
+
assert row["validation_status"] == "unvalidated"
|
| 38 |
+
assert row["validation_method"] is None
|
| 39 |
+
assert row["hf_username"] is None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def test_pending_row_preserves_sha256(monkeypatch):
|
| 43 |
+
"""Existing dedup path's row-level half: sha256 still gets stamped on the row."""
|
| 44 |
+
monkeypatch.setattr(submit, "_resolve_data_revision", lambda: "test-rev")
|
| 45 |
+
expected_hash = "f" * 64
|
| 46 |
+
row = submit._build_pending_row(
|
| 47 |
+
submission_id="sub-test-x",
|
| 48 |
+
meta=_stub_meta(),
|
| 49 |
+
blob_url="https://huggingface.co/datasets/example/sub-test-x.zip",
|
| 50 |
+
submission_sha256=expected_hash,
|
| 51 |
+
)
|
| 52 |
+
assert row["submission_sha256"] == expected_hash
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def test_pending_row_preserves_existing_metadata(monkeypatch):
|
| 56 |
+
"""Pre-Bundle-1+2 fields keep their values from meta + args.
|
| 57 |
+
|
| 58 |
+
Regression guard: a future refactor of ``_build_pending_row`` that
|
| 59 |
+
accidentally drops one of these keys would silently change the
|
| 60 |
+
schema of every row the Space writes.
|
| 61 |
+
"""
|
| 62 |
+
monkeypatch.setattr(submit, "_resolve_data_revision", lambda: "test-rev")
|
| 63 |
+
meta = _stub_meta()
|
| 64 |
+
row = submit._build_pending_row(
|
| 65 |
+
submission_id="sub-test-x",
|
| 66 |
+
meta=meta,
|
| 67 |
+
blob_url="https://example.test/sub-test-x.zip",
|
| 68 |
+
submission_sha256="0" * 64,
|
| 69 |
+
)
|
| 70 |
+
assert row["submission_id"] == "sub-test-x"
|
| 71 |
+
assert row["status"] == "pending"
|
| 72 |
+
assert row["failure_reason"] is None
|
| 73 |
+
assert row["submitter_name"] == meta["submitter_name"]
|
| 74 |
+
assert row["submission_name"] == meta["submission_name"]
|
| 75 |
+
assert row["agent_url"] == meta["agent_url"]
|
| 76 |
+
assert row["notes"] == meta["notes"]
|
| 77 |
+
assert row["submission_blob_url"] == "https://example.test/sub-test-x.zip"
|
| 78 |
+
assert row["cadgenbench_data_revision"] == "test-rev"
|
| 79 |
+
# Score-shaped fields are null on a fresh pending row.
|
| 80 |
+
for k in (
|
| 81 |
+
"aggregate_score",
|
| 82 |
+
"validity_rate",
|
| 83 |
+
"score_by_task_type",
|
| 84 |
+
"per_task_scores",
|
| 85 |
+
"per_fixture_scores",
|
| 86 |
+
"per_fixture_breakdown",
|
| 87 |
+
):
|
| 88 |
+
assert row[k] is None
|