tests/test_datasheet_completeness.py

"""PORT-02 completeness regression gate for DATASHEET.md.

Parses DATASHEET.md and asserts:
  - All 7 Gebru section headers present
  - Limitations section present
  - Reality Anchor placeholder section present (D-09)
  - Synthetic-vs-real gap is the LEAD ITEM in Limitations (D-09 verbatim)
  - Master seeds (20260501 train / 20260502 eval) cited
  - All 10 class slugs appear in the Reality Anchor table
  - Dataset size and Gebru citation are present
"""
from __future__ import annotations

import re
from pathlib import Path

DATASHEET_PATH = Path(__file__).parent.parent / "DATASHEET.md"

GEBRU_SECTIONS = [
    "## 1. Motivation",
    "## 2. Composition",
    "## 3. Collection Process",
    "## 4. Preprocessing/Cleaning/Labeling",
    "## 5. Uses",
    "## 6. Distribution",
    "## 7. Maintenance",
]

EXPECTED_CLASS_SLUGS = [
    "auth_8021x_eap_fail",
    "ap_roam_rekey_fail",
    "radius_timeout",
    "captive_portal_expiry",
    "mac_randomization_reject",
    "dhcp_lease_churn",
    "dns_resolver_fail",
    "driver_power_save_wake",
    "rf_sticky_client",
    "isp_upstream_fail",
]


def _content() -> str:
    assert DATASHEET_PATH.exists(), f"DATASHEET.md missing at {DATASHEET_PATH} (PORT-02)"
    return DATASHEET_PATH.read_text(encoding="utf-8")


def test_all_seven_gebru_sections_present():
    text = _content()
    missing = [s for s in GEBRU_SECTIONS if s not in text]
    assert not missing, f"DATASHEET.md missing Gebru sections: {missing}"


def test_limitations_section_present():
    text = _content()
    assert "## Limitations" in text, (
        "DATASHEET.md must include a Limitations section (D-09)"
    )


def test_reality_anchor_placeholder_section_present():
    text = _content()
    assert "## Reality Anchor" in text, (
        "DATASHEET.md must include a Reality Anchor section "
        "(D-09 — explicit subsection, not a TODO comment)"
    )
    assert re.search(r"pending.*Phase 4", text, re.IGNORECASE), (
        "Reality Anchor section must mark rows as 'pending — Phase 4' (D-09)"
    )


def test_synthetic_real_gap_is_lead_limitation():
    """D-09 verbatim: Limitations section LEADS with synthetic-vs-real gap."""
    text = _content()
    m = re.search(r"## Limitations(.*?)(?=^## |\Z)", text, re.DOTALL | re.MULTILINE)
    assert m, "Limitations section not found"
    limitations_block = m.group(1)
    first_sub = re.search(r"### \d+\.\s*([^\n]+)", limitations_block)
    assert first_sub, "Limitations must have numbered subsections"
    head = first_sub.group(1).lower()
    assert "synthetic" in head and "real" in head, (
        f"D-09 violation: first Limitations item must lead with synthetic-vs-real "
        f"gap; got: {first_sub.group(1)}"
    )


def test_master_seeds_cited():
    text = _content()
    assert "20260501" in text, "DATASHEET.md must cite MASTER_TRAIN_SEED=20260501"
    assert "20260502" in text, "DATASHEET.md must cite MASTER_EVAL_SEED=20260502"


def test_make_synth_command_documented():
    text = _content()
    assert "make synth" in text, (
        "DATASHEET.md must document the `make synth` regeneration command (D-08)"
    )


def test_all_ten_class_slugs_in_reality_anchor():
    text = _content()
    m = re.search(r"## Reality Anchor(.*?)(?=^## |\Z)", text, re.DOTALL | re.MULTILINE)
    assert m, "Reality Anchor section not found"
    ra_block = m.group(1)
    missing = [c for c in EXPECTED_CLASS_SLUGS if c not in ra_block]
    assert not missing, f"Reality Anchor table missing class slugs: {missing}"


def test_dataset_size_documented():
    text = _content()
    has_total = any(s in text for s in ("100,000", "100000", "100_000", "100k"))
    has_per_class = any(s in text for s in ("10,000", "10000", "10_000", "10k"))
    assert has_total, "DATASHEET.md must document the 100k total sample size (D-08)"
    assert has_per_class, "DATASHEET.md must document the 10k-per-class breakdown (D-07)"


def test_gebru_citation():
    text = _content()
    assert "Gebru" in text, "DATASHEET.md must cite Gebru et al."
    assert "1803.09010" in text or "Datasheets for Datasets" in text, (
        "DATASHEET.md must reference the Datasheets for Datasets paper"
    )