Michael Rabinovich commited on
Commit
a533dd2
·
1 Parent(s): 36546fc

submit: default validation + hf_username fields on the pending row

Browse files

Bundle 1+2 C4. `_build_pending_row` adds three new keys matching the
schema bump from C1, in the schema.md column order:

- validation_status = "unvalidated" (every new row starts in the
unvalidated tier; maintainers promote via the dataset repo after
methodology review).
- validation_method = None (set by the maintainer at promotion time
to one of code | traces | api | manual).
- hf_username = None (placeholder; C10 plumbs through the value
from gradio.OAuthProfile.username once the LoginButton lands).

Pre-existing fields, submitted_at stamping, the cadgenbench version
+ data revision tags, the score-shaped null fields, the blob URL,
and the sha256 are untouched. Dedup gate still pivots on
submission_sha256 exactly as before.

tests/test_submit.py covers the C4 acceptance:

- new-row defaults: validation_status is "unvalidated",
validation_method + hf_username are None;
- sha256 path passthrough: the hash argument lands on the row's
submission_sha256 key intact (existing dedup gate's row-level
half is unchanged);
- existing-metadata regression guard: submission_id, status,
submitter_name, submission_name, agent_url, notes,
submission_blob_url, cadgenbench_data_revision, and every
score-shaped null field all keep their pre-C4 values.

`_resolve_data_revision` is monkeypatched so the suite has zero
Hub I/O; cadgenbench.__version__ + datetime.now both stay live
(pure / process-local). 7/7 unit tests green locally (3 new + 4
existing leaderboard).

Files changed (2) hide show
  1. submit.py +9 -0
  2. tests/test_submit.py +88 -0
submit.py CHANGED
@@ -401,6 +401,12 @@ def _build_pending_row(
401
  metadata + ``status: pending`` + ``submission_blob_url`` +
402
  ``submission_sha256``; every score-shaped field is ``null`` until
403
  the worker flips the row.
 
 
 
 
 
 
404
  """
405
  return {
406
  "submission_id": submission_id,
@@ -421,6 +427,9 @@ def _build_pending_row(
421
  "per_fixture_breakdown": None,
422
  "submission_blob_url": blob_url,
423
  "submission_sha256": submission_sha256,
 
 
 
424
  }
425
 
426
 
 
401
  metadata + ``status: pending`` + ``submission_blob_url`` +
402
  ``submission_sha256``; every score-shaped field is ``null`` until
403
  the worker flips the row.
404
+
405
+ Validation-tier fields default per the validation-policy decision
406
+ doc: ``validation_status: "unvalidated"`` (maintainers promote
407
+ post-eval), ``validation_method: None``. ``hf_username`` is also
408
+ ``None`` here; C10 wires it through from ``gr.OAuthProfile`` once
409
+ the LoginButton lands.
410
  """
411
  return {
412
  "submission_id": submission_id,
 
427
  "per_fixture_breakdown": None,
428
  "submission_blob_url": blob_url,
429
  "submission_sha256": submission_sha256,
430
+ "validation_status": "unvalidated",
431
+ "validation_method": None,
432
+ "hf_username": None,
433
  }
434
 
435
 
tests/test_submit.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Unit tests for the submit-tab pending-row builder.
2
+
3
+ C4 contract: ``_build_pending_row`` defaults the three Bundle 1+2
4
+ schema fields (``validation_status="unvalidated"``,
5
+ ``validation_method=None``, ``hf_username=None``) and keeps the
6
+ existing metadata + sha256 fields intact.
7
+
8
+ Hub I/O (``_resolve_data_revision`` reads from
9
+ ``HfApi.dataset_info``) is monkeypatched out, so the suite has zero
10
+ network traffic.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import submit
15
+
16
+
17
+ def _stub_meta() -> dict:
18
+ """Minimum meta.json shape that survives ``_load_and_validate_meta``."""
19
+ return {
20
+ "submitter_name": "team-test",
21
+ "submission_name": "Stub Agent v1",
22
+ "agent_url": "https://github.com/example/stub-agent",
23
+ "notes": "test row, not a real submission",
24
+ "agree_to_publish": True,
25
+ }
26
+
27
+
28
+ def test_pending_row_defaults_new_fields(monkeypatch):
29
+ """Three Bundle 1+2 fields land with their schema defaults."""
30
+ monkeypatch.setattr(submit, "_resolve_data_revision", lambda: "test-rev")
31
+ row = submit._build_pending_row(
32
+ submission_id="sub-test-x",
33
+ meta=_stub_meta(),
34
+ blob_url="https://huggingface.co/datasets/example/sub-test-x.zip",
35
+ submission_sha256="a" * 64,
36
+ )
37
+ assert row["validation_status"] == "unvalidated"
38
+ assert row["validation_method"] is None
39
+ assert row["hf_username"] is None
40
+
41
+
42
+ def test_pending_row_preserves_sha256(monkeypatch):
43
+ """Existing dedup path's row-level half: sha256 still gets stamped on the row."""
44
+ monkeypatch.setattr(submit, "_resolve_data_revision", lambda: "test-rev")
45
+ expected_hash = "f" * 64
46
+ row = submit._build_pending_row(
47
+ submission_id="sub-test-x",
48
+ meta=_stub_meta(),
49
+ blob_url="https://huggingface.co/datasets/example/sub-test-x.zip",
50
+ submission_sha256=expected_hash,
51
+ )
52
+ assert row["submission_sha256"] == expected_hash
53
+
54
+
55
+ def test_pending_row_preserves_existing_metadata(monkeypatch):
56
+ """Pre-Bundle-1+2 fields keep their values from meta + args.
57
+
58
+ Regression guard: a future refactor of ``_build_pending_row`` that
59
+ accidentally drops one of these keys would silently change the
60
+ schema of every row the Space writes.
61
+ """
62
+ monkeypatch.setattr(submit, "_resolve_data_revision", lambda: "test-rev")
63
+ meta = _stub_meta()
64
+ row = submit._build_pending_row(
65
+ submission_id="sub-test-x",
66
+ meta=meta,
67
+ blob_url="https://example.test/sub-test-x.zip",
68
+ submission_sha256="0" * 64,
69
+ )
70
+ assert row["submission_id"] == "sub-test-x"
71
+ assert row["status"] == "pending"
72
+ assert row["failure_reason"] is None
73
+ assert row["submitter_name"] == meta["submitter_name"]
74
+ assert row["submission_name"] == meta["submission_name"]
75
+ assert row["agent_url"] == meta["agent_url"]
76
+ assert row["notes"] == meta["notes"]
77
+ assert row["submission_blob_url"] == "https://example.test/sub-test-x.zip"
78
+ assert row["cadgenbench_data_revision"] == "test-rev"
79
+ # Score-shaped fields are null on a fresh pending row.
80
+ for k in (
81
+ "aggregate_score",
82
+ "validity_rate",
83
+ "score_by_task_type",
84
+ "per_task_scores",
85
+ "per_fixture_scores",
86
+ "per_fixture_breakdown",
87
+ ):
88
+ assert row[k] is None