Spaces:
Running
Running
| import json | |
| from pathlib import Path | |
| import pytest | |
| from demo.app import run_verification | |
| from demo.examples import EXAMPLE_NAMES, example_metadata | |
| from demo.faultless import NAMES as COMPLETENESS_NAMES | |
| def test_faultless_frontend_stream(tmp_path, compiler, name, example_rows): | |
| rows = example_rows | |
| args = rows[EXAMPLE_NAMES.index(name)].copy() | |
| args[8] = compiler | |
| seed_count = args[5] | |
| updates = list(run_verification(*args, report_directory=tmp_path / 'reports')) | |
| report = json.loads(Path(updates[-1][3]).read_text()) | |
| assert len(updates) == seed_count + 1 | |
| assert report['completed'] == seed_count | |
| expected = example_metadata(name)['expected_verdict'] | |
| if name == 'faultless-byte-reinterpretation' and compiler == 'clang': | |
| # The original is built with gcc, so the escaped frames differ in layout. | |
| expected = 'inconclusive' | |
| assert all(r['reason'] == 'escaped_frame_differs' for r in report['results']) | |
| assert report['verdict'] == expected | |
| assert [r['seed'] for r in report['results']] == list(range(seed_count)) | |
| if name == 'faultless-linked-list': | |
| assert report['counts']['mismatch'] == seed_count | |
| elif name == 'faultless-deep-pointer': | |
| assert report['counts']['mismatch'] == seed_count | |
| assert all(r['divergence'] == 'call[1]' for r in report['results']) | |
| elif name == 'faultless-heap-loop': | |
| assert report['counts']['agree'] > 0 | |
| assert report['counts']['mismatch'] > 0 | |
| elif name == 'faultless-memory-dependency': | |
| assert report['counts']['agree'] == seed_count | |
| elif name == 'faultless-byte-reinterpretation': | |
| assert report['counts'][expected] == seed_count | |
| elif name in COMPLETENESS_NAMES: | |
| assert report['counts'][expected] == seed_count | |
| if name == 'faultless-my-putchar-count': | |
| assert all(r['reason'] == 'escaped_frame_differs' for r in report['results']) | |
| else: | |
| assert report['counts']['mismatch'] == seed_count | |
| assert all(r['divergence'] == 'call[0]' for r in report['results']) | |