Michael Rabinovich commited on
Commit ·
007a50e
1
Parent(s): 47c86cf
tests: cover the admin gate and promote/demote helpers
Browse filesfour unit tests for admin.py: the CADGENBENCH_ADMINS gate (in-set,
out-of-set, logged-out, empty set), promote and demote happy paths,
and idempotent re-promotion. the results.jsonl read and write that
_hub_rmw_results drives are mocked, so the suite makes no hub calls.
- tests/test_admin.py +120 -0
tests/test_admin.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Unit tests for the admin promote / demote helpers.
|
| 2 |
+
|
| 3 |
+
C5 contract: the admin gate distinguishes in-set from out-of-set
|
| 4 |
+
users, ``promote_row`` and ``demote_row`` flip the two validation
|
| 5 |
+
fields on the right row, and a repeat promotion is idempotent. Every
|
| 6 |
+
``results.jsonl`` read and write is mocked, so the suite makes zero
|
| 7 |
+
Hub calls.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
from types import SimpleNamespace
|
| 14 |
+
|
| 15 |
+
import pytest
|
| 16 |
+
|
| 17 |
+
# submit.py kicks off a Hub-touching stuck-pending sweep at import.
|
| 18 |
+
# Disable it before importing (admin imports submit) so running this
|
| 19 |
+
# file in isolation stays offline.
|
| 20 |
+
os.environ.setdefault("CADGENBENCH_DISABLE_BOOT_SWEEP", "1")
|
| 21 |
+
|
| 22 |
+
import admin # noqa: E402
|
| 23 |
+
import submit # noqa: E402
|
| 24 |
+
|
| 25 |
+
SEED_ROWS = [
|
| 26 |
+
{
|
| 27 |
+
"submission_id": "alpha",
|
| 28 |
+
"validation_status": "unvalidated",
|
| 29 |
+
"validation_method": None,
|
| 30 |
+
"aggregate_score": 0.5,
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"submission_id": "beta",
|
| 34 |
+
"validation_status": "validated",
|
| 35 |
+
"validation_method": "code",
|
| 36 |
+
"aggregate_score": 0.9,
|
| 37 |
+
},
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _jsonl(rows: list[dict]) -> str:
|
| 42 |
+
return "\n".join(json.dumps(r) for r in rows) + "\n"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _row(rows: list[dict], submission_id: str) -> dict:
|
| 46 |
+
return next(r for r in rows if r["submission_id"] == submission_id)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
@pytest.fixture
|
| 50 |
+
def hub(monkeypatch):
|
| 51 |
+
"""Mock the results.jsonl read + write that ``_hub_rmw_results`` drives.
|
| 52 |
+
|
| 53 |
+
``state["rows"]`` starts as a copy of :data:`SEED_ROWS` and is
|
| 54 |
+
replaced by whatever bytes the helper hands to ``upload_file``,
|
| 55 |
+
re-parsed back into dicts. ``state["uploads"]`` counts the writes
|
| 56 |
+
so a test can assert how many commits a call produced.
|
| 57 |
+
"""
|
| 58 |
+
state: dict = {"rows": [dict(r) for r in SEED_ROWS], "uploads": 0}
|
| 59 |
+
|
| 60 |
+
def fake_download() -> str:
|
| 61 |
+
return _jsonl(state["rows"])
|
| 62 |
+
|
| 63 |
+
def fake_upload(*, path_or_fileobj, **kwargs) -> None:
|
| 64 |
+
body = (
|
| 65 |
+
path_or_fileobj.decode("utf-8")
|
| 66 |
+
if isinstance(path_or_fileobj, bytes)
|
| 67 |
+
else path_or_fileobj
|
| 68 |
+
)
|
| 69 |
+
state["rows"] = [
|
| 70 |
+
json.loads(line) for line in body.splitlines() if line.strip()
|
| 71 |
+
]
|
| 72 |
+
state["uploads"] += 1
|
| 73 |
+
|
| 74 |
+
monkeypatch.setattr(submit, "_download_results_jsonl", fake_download)
|
| 75 |
+
monkeypatch.setattr(submit._HF_API, "upload_file", fake_upload)
|
| 76 |
+
return state
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def test_admin_gate_in_set_vs_out_of_set(monkeypatch):
|
| 80 |
+
"""is_admin admits logged-in users in the set, rejects everyone else."""
|
| 81 |
+
monkeypatch.setenv("CADGENBENCH_ADMINS", "michaelr27, lvwerra")
|
| 82 |
+
assert admin.is_admin(SimpleNamespace(username="michaelr27")) is True
|
| 83 |
+
assert admin.is_admin(SimpleNamespace(username="lvwerra")) is True
|
| 84 |
+
assert admin.is_admin(SimpleNamespace(username="someone-else")) is False
|
| 85 |
+
# Logged-out is never admin.
|
| 86 |
+
assert admin.is_admin(None) is False
|
| 87 |
+
# Empty / unset variable means no one is admin.
|
| 88 |
+
monkeypatch.delenv("CADGENBENCH_ADMINS", raising=False)
|
| 89 |
+
assert admin.is_admin(SimpleNamespace(username="michaelr27")) is False
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def test_promote_happy_path(hub):
|
| 93 |
+
"""Promoting an unvalidated row sets status + method, leaves others alone."""
|
| 94 |
+
admin.promote_row("alpha", "manual")
|
| 95 |
+
promoted = _row(hub["rows"], "alpha")
|
| 96 |
+
assert promoted["validation_status"] == "validated"
|
| 97 |
+
assert promoted["validation_method"] == "manual"
|
| 98 |
+
# The other row is untouched.
|
| 99 |
+
assert _row(hub["rows"], "beta")["validation_method"] == "code"
|
| 100 |
+
assert hub["uploads"] == 1
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def test_demote_happy_path(hub):
|
| 104 |
+
"""Demoting a validated row clears the method and flips status back."""
|
| 105 |
+
admin.demote_row("beta")
|
| 106 |
+
demoted = _row(hub["rows"], "beta")
|
| 107 |
+
assert demoted["validation_status"] == "unvalidated"
|
| 108 |
+
assert demoted["validation_method"] is None
|
| 109 |
+
assert hub["uploads"] == 1
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def test_promote_idempotent(hub):
|
| 113 |
+
"""Re-promoting an already-validated row lands the same state."""
|
| 114 |
+
admin.promote_row("beta", "code")
|
| 115 |
+
once = dict(_row(hub["rows"], "beta"))
|
| 116 |
+
assert once["validation_status"] == "validated"
|
| 117 |
+
assert once["validation_method"] == "code"
|
| 118 |
+
# Second identical promotion produces an identical row.
|
| 119 |
+
admin.promote_row("beta", "code")
|
| 120 |
+
assert _row(hub["rows"], "beta") == once
|