"""PORT-02 completeness regression gate for DATASHEET.md. Parses DATASHEET.md and asserts: - All 7 Gebru section headers present - Limitations section present - Reality Anchor placeholder section present (D-09) - Synthetic-vs-real gap is the LEAD ITEM in Limitations (D-09 verbatim) - Master seeds (20260501 train / 20260502 eval) cited - All 10 class slugs appear in the Reality Anchor table - Dataset size and Gebru citation are present """ from __future__ import annotations import re from pathlib import Path DATASHEET_PATH = Path(__file__).parent.parent / "DATASHEET.md" GEBRU_SECTIONS = [ "## 1. Motivation", "## 2. Composition", "## 3. Collection Process", "## 4. Preprocessing/Cleaning/Labeling", "## 5. Uses", "## 6. Distribution", "## 7. Maintenance", ] EXPECTED_CLASS_SLUGS = [ "auth_8021x_eap_fail", "ap_roam_rekey_fail", "radius_timeout", "captive_portal_expiry", "mac_randomization_reject", "dhcp_lease_churn", "dns_resolver_fail", "driver_power_save_wake", "rf_sticky_client", "isp_upstream_fail", ] def _content() -> str: assert DATASHEET_PATH.exists(), f"DATASHEET.md missing at {DATASHEET_PATH} (PORT-02)" return DATASHEET_PATH.read_text(encoding="utf-8") def test_all_seven_gebru_sections_present(): text = _content() missing = [s for s in GEBRU_SECTIONS if s not in text] assert not missing, f"DATASHEET.md missing Gebru sections: {missing}" def test_limitations_section_present(): text = _content() assert "## Limitations" in text, ( "DATASHEET.md must include a Limitations section (D-09)" ) def test_reality_anchor_placeholder_section_present(): text = _content() assert "## Reality Anchor" in text, ( "DATASHEET.md must include a Reality Anchor section " "(D-09 — explicit subsection, not a TODO comment)" ) assert re.search(r"pending.*Phase 4", text, re.IGNORECASE), ( "Reality Anchor section must mark rows as 'pending — Phase 4' (D-09)" ) def test_synthetic_real_gap_is_lead_limitation(): """D-09 verbatim: Limitations section LEADS with synthetic-vs-real gap.""" text = _content() m = re.search(r"## Limitations(.*?)(?=^## |\Z)", text, re.DOTALL | re.MULTILINE) assert m, "Limitations section not found" limitations_block = m.group(1) first_sub = re.search(r"### \d+\.\s*([^\n]+)", limitations_block) assert first_sub, "Limitations must have numbered subsections" head = first_sub.group(1).lower() assert "synthetic" in head and "real" in head, ( f"D-09 violation: first Limitations item must lead with synthetic-vs-real " f"gap; got: {first_sub.group(1)}" ) def test_master_seeds_cited(): text = _content() assert "20260501" in text, "DATASHEET.md must cite MASTER_TRAIN_SEED=20260501" assert "20260502" in text, "DATASHEET.md must cite MASTER_EVAL_SEED=20260502" def test_make_synth_command_documented(): text = _content() assert "make synth" in text, ( "DATASHEET.md must document the `make synth` regeneration command (D-08)" ) def test_all_ten_class_slugs_in_reality_anchor(): text = _content() m = re.search(r"## Reality Anchor(.*?)(?=^## |\Z)", text, re.DOTALL | re.MULTILINE) assert m, "Reality Anchor section not found" ra_block = m.group(1) missing = [c for c in EXPECTED_CLASS_SLUGS if c not in ra_block] assert not missing, f"Reality Anchor table missing class slugs: {missing}" def test_dataset_size_documented(): text = _content() has_total = any(s in text for s in ("100,000", "100000", "100_000", "100k")) has_per_class = any(s in text for s in ("10,000", "10000", "10_000", "10k")) assert has_total, "DATASHEET.md must document the 100k total sample size (D-08)" assert has_per_class, "DATASHEET.md must document the 10k-per-class breakdown (D-07)" def test_gebru_citation(): text = _content() assert "Gebru" in text, "DATASHEET.md must cite Gebru et al." assert "1803.09010" in text or "Datasheets for Datasets" in text, ( "DATASHEET.md must reference the Datasheets for Datasets paper" )