tests/test_artifacts.py

"""Every committed artifact matches the schema its generator declares."""
import pytest

from common.artifacts import REGISTRY
from conftest import REPO, load

# Artifacts whose producing sweep was never committed, so provenance.generator
# is null. A new entry here is a new gap and has to be added deliberately.
KNOWN_GAPS = {
    'eval_tight_fpr.json',
    'discovery/dim_selection.json',
    'discovery/dim48_characterization.json',
    'discovery/prop_specificity.json',
    'discovery/prop_image_manifest.json',
    'discovery/variant_leaderboard.json',
}


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_exists(rel):
    assert (REPO / rel).exists(), f'{rel} is registered but missing'


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_has_provenance(rel):
    doc = load(rel)
    assert 'provenance' in doc, f'{rel} has no provenance block'
    assert next(iter(doc)) == 'provenance', f'{rel} does not open with its provenance block'


@pytest.mark.parametrize('rel', sorted(r for r in REGISTRY if REGISTRY[r].payload_keys))
def test_artifact_payload_keys(rel):
    spec = REGISTRY[rel]
    got = tuple(k for k in load(rel) if k != 'provenance')
    assert got == spec.payload_keys, f'{rel} payload keys {got} != declared {spec.payload_keys}'


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_generator(rel):
    spec = REGISTRY[rel]
    generator = load(rel)['provenance']['generator']
    assert generator in (spec.generator, None), \
        f'{rel} claims generator {generator!r}, registry says {spec.generator!r}'


@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_pool(rel):
    spec = REGISTRY[rel]
    pool = load(rel)['provenance'].get('pool')
    if spec.pool is not None and pool is not None:
        assert pool == spec.pool, f'{rel} names pool {pool!r}, registry says {spec.pool!r}'


def test_known_gaps_are_exactly_the_ungenerated_artifacts():
    ungenerated = {rel for rel in REGISTRY
                   if load(rel)['provenance']['generator'] is None}
    assert ungenerated == KNOWN_GAPS


def test_classifier_hashes_are_current():
    """Every artifact naming a classifier config records that file's current hash."""
    from common.artifacts import sha256_of
    for rel in sorted(REGISTRY):
        p = load(rel)['provenance']
        if 'classifier' not in p:
            continue
        assert p['classifier_sha256'] == sha256_of(REPO / p['classifier']), \
            f'{rel} pins a stale hash for {p["classifier"]}'