File size: 2,531 Bytes
f5498f9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | """Every committed artifact matches the schema its generator declares."""
import pytest
from common.artifacts import REGISTRY
from conftest import REPO, load
# Artifacts whose producing sweep was never committed, so provenance.generator
# is null. A new entry here is a new gap and has to be added deliberately.
KNOWN_GAPS = {
'eval_tight_fpr.json',
'discovery/dim_selection.json',
'discovery/dim48_characterization.json',
'discovery/prop_specificity.json',
'discovery/prop_image_manifest.json',
'discovery/variant_leaderboard.json',
}
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_exists(rel):
assert (REPO / rel).exists(), f'{rel} is registered but missing'
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_has_provenance(rel):
doc = load(rel)
assert 'provenance' in doc, f'{rel} has no provenance block'
assert next(iter(doc)) == 'provenance', f'{rel} does not open with its provenance block'
@pytest.mark.parametrize('rel', sorted(r for r in REGISTRY if REGISTRY[r].payload_keys))
def test_artifact_payload_keys(rel):
spec = REGISTRY[rel]
got = tuple(k for k in load(rel) if k != 'provenance')
assert got == spec.payload_keys, f'{rel} payload keys {got} != declared {spec.payload_keys}'
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_generator(rel):
spec = REGISTRY[rel]
generator = load(rel)['provenance']['generator']
assert generator in (spec.generator, None), \
f'{rel} claims generator {generator!r}, registry says {spec.generator!r}'
@pytest.mark.parametrize('rel', sorted(REGISTRY))
def test_artifact_pool(rel):
spec = REGISTRY[rel]
pool = load(rel)['provenance'].get('pool')
if spec.pool is not None and pool is not None:
assert pool == spec.pool, f'{rel} names pool {pool!r}, registry says {spec.pool!r}'
def test_known_gaps_are_exactly_the_ungenerated_artifacts():
ungenerated = {rel for rel in REGISTRY
if load(rel)['provenance']['generator'] is None}
assert ungenerated == KNOWN_GAPS
def test_classifier_hashes_are_current():
"""Every artifact naming a classifier config records that file's current hash."""
from common.artifacts import sha256_of
for rel in sorted(REGISTRY):
p = load(rel)['provenance']
if 'classifier' not in p:
continue
assert p['classifier_sha256'] == sha256_of(REPO / p['classifier']), \
f'{rel} pins a stale hash for {p["classifier"]}'
|