| """Prepare or verify a local GPL-2.0 v1 release. Never trains, converts or uploads.""" |
| import argparse |
| import importlib.util |
| import json |
| import re |
| import shutil |
| import subprocess |
| import sys |
| from datetime import datetime, timezone |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(ROOT / 'src')) |
|
|
|
|
| def verifier(): |
| spec = importlib.util.spec_from_file_location('vimeml_release_verifier', ROOT / 'templates/huggingface/verify_release.py') |
| module = importlib.util.module_from_spec(spec) |
| spec.loader.exec_module(module) |
| return module |
|
|
|
|
| def write_json(path, value): |
| path.parent.mkdir(parents=True, exist_ok=True) |
| path.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + '\n', encoding='utf-8', newline='\n') |
|
|
|
|
| def git(*args): |
| return subprocess.check_output(['git', *args], cwd=ROOT).decode('utf-8').strip() |
|
|
|
|
| def copy_checked(source, target, digest=None): |
| check = verifier() |
| source = Path(source) |
| if source.is_symlink() or not source.is_file(): |
| raise ValueError(f'Expected a regular source file: {source}') |
| expected = digest or check.sha(source) |
| target.parent.mkdir(parents=True, exist_ok=True) |
| if target.exists(): |
| raise ValueError(f'Refusing to overwrite: {target}') |
| shutil.copyfile(source, target) |
| if check.sha(target) != expected: |
| raise ValueError(f'Copy checksum mismatch: {target}') |
|
|
|
|
| def evaluation_summary(bundle_manifest_sha, coreml_manifest_sha): |
| from vimeml.benchmarks.evaluate_ajimee import load_export, summarize |
| from vimeml.deployment.bundle import read_json |
| from vimeml.deployment.validation import compare_rows |
| sha = verifier().sha |
| hashes, results = {}, {} |
|
|
| def read(path): |
| hashes[path.relative_to(ROOT).as_posix()] = sha(path) |
| return read_json(path) |
|
|
| def rows(path, metadata): |
| expected = metadata['files_sha256']['scores.jsonl'] |
| if sha(path) != expected: |
| raise ValueError(f'Score file identity mismatch: {path}') |
| hashes[path.relative_to(ROOT).as_posix()] = expected |
| return [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines()] |
|
|
| for role, benchmark, baseline in [('dev', 'ime-dev-v2', 'tiny-ja-v1-dev-v2-scores'), |
| ('ajimee', 'ajimee-jwtd-v2-v1', 'tiny-ja-v1-ajimee')]: |
| directory = ROOT / f'outputs/deployment/conservative-int8-b32-{role}-v1' |
| report = read(directory / 'metrics.json') |
| if (report['model']['coreml_manifest_sha256'] != coreml_manifest_sha or |
| report['model']['bundle_manifest_sha256'] != bundle_manifest_sha): |
| raise ValueError('Saved evaluation uses a different model.') |
| current = rows(directory / 'scores.jsonl', report) |
| metrics = summarize(current) |
| if metrics != report['metrics']: |
| raise ValueError('Saved metrics do not match their score rows.') |
| frozen, provenance, inputs = load_export(ROOT / 'artifacts/benchmarks' / benchmark) |
| if frozen != report['benchmark_manifest'] or provenance != report['export_provenance'] or len(inputs) != len(current): |
| raise ValueError('Frozen candidate export identity changed.') |
| for left, right in zip(inputs, current): |
| if any(left[key] != right[key] for key in ('id', 'query', 'left_context', 'answers', 'candidates')): |
| raise ValueError('Saved candidate/input mismatch.') |
| prior_directory = ROOT / 'outputs/ime-eval' / baseline |
| prior_path = prior_directory / ('manifest.json' if (prior_directory / 'manifest.json').exists() else 'metrics.json') |
| prior_report = read(prior_path) |
| if any(prior_report['model'][key] != report['model'][key] |
| for key in ('checkpoint_sha256', 'tokenizer_sha256')): |
| raise ValueError('FP32 baseline belongs to different frozen weights or tokenizer.') |
| prior = rows(prior_directory / 'scores.jsonl', prior_report) |
| fp32_metrics = summarize(prior) |
| comparison = compare_rows(prior, current) |
| reported_before = prior |
| if role == 'dev': |
| source = ROOT / 'outputs/deployment/conservative-dev-v3' |
| source_metadata = read(source / 'metrics.json') |
| reported_before = rows(source / 'scores.jsonl', source_metadata) |
| if compare_rows(reported_before, current) != report['comparison']: |
| raise ValueError('Saved comparison cannot be reproduced from its actual baseline.') |
| results[role] = {'original_order': metrics['all']['azookey'], |
| 'fp32_lm': fp32_metrics['all']['lm_context_sum'], 'int8_lm': metrics['all']['lm_context_sum'], |
| 'windows_fp32_comparison': {k: v for k, v in comparison.items() if k != 'changes'}, |
| 'original_report_comparison_baseline': 'conservative Core ML' if role == 'dev' else 'Windows FP32', |
| 'original_report_comparison': {k: v for k, v in report['comparison'].items() if k != 'changes'}, |
| 'benchmark_source_sha256': frozen['source_sha256'], 'candidate_export_sha256': provenance['files_sha256']['azookey-candidates.json']} |
| alignment = read(ROOT / 'outputs/deployment/conservative-int8-b32-alignment-v1/alignment.json') |
| if alignment['coreml_manifest_sha256'] != coreml_manifest_sha: |
| raise ValueError('Alignment belongs to a different model.') |
| distribution = read(ROOT / 'outputs/deployment/conservative-int8-b32-distribution-v1/report.json') |
| training = read(ROOT / 'artifacts/models/tiny-ja-v1/summary.json') |
| return {'format': 'vimeml_public_evaluation_v1', 'bundle_manifest_sha256': bundle_manifest_sha, |
| 'coreml_manifest_sha256': coreml_manifest_sha, 'training': { |
| 'prediction_targets': training['total_trained_tokens'], 'data_passes': training['completed_data_passes'], |
| 'full_validation': training['full_validation']['best'], 'test_used_for_selection': False}, |
| 'ranking': results, 'alignment': {'strict_passed': alignment['passed'], |
| 'max_logit_abs': max(row['max_abs'] for row in alignment['logits']), |
| 'invariants': alignment['invariants']}, |
| 'distribution': {'original_report': distribution, |
| 'aggregation': 'Mean of per-example position means; not corpus token-weighted.', |
| 'content_token_limit': 127, 'truncation_count_recorded': False, |
| 'original_report_has_full_identity_metadata': False}, |
| 'source_reports_sha256': hashes, |
| 'scope': 'Offline fixed pools and saved Mac observations; not app-wide accuracy or a Windows Core ML runtime rerun.'} |
|
|
|
|
| def prepare(output, repo_id='Voltline/vimeml-tiny-ja-v1'): |
| output = Path(output).resolve() |
| if output.exists(): |
| raise ValueError('Output exists; choose a new release version.') |
| if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo_id): |
| raise ValueError('Use an OWNER/REPO model repository ID.') |
| if git('status', '--porcelain'): |
| raise ValueError('Commit the reviewed source on the current branch before preparing a release.') |
| from vimeml.deployment.bundle import verify_bundle |
| from vimeml.deployment.coreml import verify_package |
| check = verifier() |
| bundle = ROOT / 'artifacts/deployment/tiny-ja-v1-inference-v1' |
| model = ROOT / 'artifacts/deployment/tiny-ja-v1-conservative-int8-b32-v1' |
| metadata = verify_bundle(bundle) |
| coreml = verify_package(model) |
| bundle_sha, coreml_sha = check.sha(bundle / 'manifest.json'), check.sha(model / 'manifest.json') |
| if (metadata['parameter_count'] != 7386624 or coreml['kind'] != 'linear8_fp32_compute' or |
| coreml['minimum_ios'] != 18 or coreml['package_bytes'] != 8077801 or |
| coreml['bundle']['bundle_manifest_sha256'] != bundle_sha): |
| raise ValueError('Expected the reviewed frozen v1/INT8 package.') |
| summary = evaluation_summary(bundle_sha, coreml_sha) |
| for role, expected in [('dev', (137, 122, 134)), ('ajimee', (200, 124, 151))]: |
| for variant in ('fp32_lm', 'int8_lm'): |
| actual = summary['ranking'][role][variant] |
| if tuple(actual[key] for key in ('cases', 'top1_correct', 'top5_correct')) != expected: |
| raise ValueError('Saved metrics differ from this v1 model card; review before publishing.') |
| commit = git('rev-parse', 'HEAD') |
| output.mkdir(parents=True, exist_ok=False) |
| for name in [*metadata['files'], 'manifest.json']: |
| copy_checked(bundle / name, output / 'inference' / name) |
| for name, entry in coreml['package_files'].items(): |
| copy_checked(model / 'model.mlpackage' / name, |
| output / 'coreml/ios18-int8-block32/model.mlpackage' / name, entry['sha256']) |
| copy_checked(model / 'manifest.json', output / 'coreml/ios18-int8-block32/manifest.json') |
| |
| source_files = subprocess.check_output(['git', 'ls-files', '-z', '--', 'src', 'scripts', 'configs', |
| 'templates/huggingface', 'requirements.txt', 'LICENSE'], cwd=ROOT).decode('utf-8').split('\0') |
| for name in filter(None, source_files): |
| copy_checked(ROOT / name, output / 'source' / name) |
| (output / 'source/README.md').write_text( |
| f'VimeML source/configuration from Git commit {commit}. GPL-2.0; see LICENSE.\n' |
| 'The FP32 loader checks the original model and deployment code fingerprints.\n' |
| 'Corpus texts, annotations, optimizer state and the separate Vime client are not included.\n', encoding='utf-8', newline='\n') |
| copy_checked(ROOT / 'LICENSE', output / 'LICENSE') |
| for name in ('infer.py', 'verify_release.py'): |
| copy_checked(ROOT / 'templates/huggingface' / name, output / name) |
| card = (ROOT / 'templates/huggingface/README.md').read_text(encoding='utf-8') |
| for key, value in {'SOURCE_COMMIT': commit, 'BUNDLE_SHA': bundle_sha, 'COREML_SHA': coreml_sha, |
| 'TOKENIZER_SHA': metadata['files']['tokenizer.model']['sha256'], |
| 'CHECKPOINT_SHA': metadata['checkpoint_sha256']}.items(): |
| card = card.replace(f'@@{key}@@', value) |
| if '@@' in card: |
| raise ValueError('Unresolved model card field.') |
| (output / 'README.md').write_text(card, encoding='utf-8', newline='\n') |
| write_json(output / 'evaluation/summary.json', summary) |
| files = {p.relative_to(output).as_posix(): {'bytes': p.stat().st_size, 'sha256': check.sha(p)} |
| for p in sorted(output.rglob('*')) if p.is_file()} |
| release = {'format': 'vimeml_hf_release_v1', 'repo_id': repo_id, 'license': 'gpl-2.0', |
| 'source_repository': 'https://github.com/Voltline/VimeML', 'source_commit': commit, |
| 'created_utc': datetime.now(timezone.utc).isoformat(), 'bundle_manifest_sha256': bundle_sha, |
| 'coreml_manifest_sha256': coreml_sha, 'files': files, |
| 'policy': 'Local preparation only. No training, conversion, compression, device access or upload.'} |
| write_json(output / 'RELEASE.json', release) |
| sums = {**{name: entry['sha256'] for name, entry in files.items()}, 'RELEASE.json': check.sha(output / 'RELEASE.json')} |
| (output / 'SHA256SUMS.txt').write_text(''.join(f'{digest} {name}\n' for name, digest in sorted(sums.items())), |
| encoding='utf-8', newline='\n') |
| _, verified = check.verify(output) |
| print(json.dumps({'release': str(output), 'repo_id': repo_id, 'source_commit': commit, |
| 'verified_files': len(verified), 'bytes': sum((output / name).stat().st_size for name in verified)}, indent=2)) |
| return release |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| sub = parser.add_subparsers(dest='command', required=True) |
| build = sub.add_parser('prepare', help='Create a fresh local release from the committed source and frozen artifacts.') |
| build.add_argument('--output', type=Path, required=True) |
| build.add_argument('--repo-id', default='Voltline/vimeml-tiny-ja-v1') |
| verify = sub.add_parser('verify', help='Read-only checksum/inventory verification.') |
| verify.add_argument('--release', type=Path, required=True) |
| args = parser.parse_args() |
| if args.command == 'prepare': |
| prepare(args.output, args.repo_id) |
| else: |
| manifest, files = verifier().verify(args.release) |
| print(f"Verified {len(files)} release files for {manifest['repo_id']}") |
|
|
|
|
| if __name__ == '__main__': |
| main() |
|
|
