File size: 5,520 Bytes
d35e174 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | """Build an explicit Timestral release inventory and reject stale validation."""
import hashlib
import json
import shutil
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
VARIANTS=("5M","28M","123M","209M")
def sha256(path):
h=hashlib.sha256()
with Path(path).open('rb') as f:
for block in iter(lambda:f.read(8*1024*1024),b''): h.update(block)
return h.hexdigest()
def excluded(path):
return any(p.startswith('._') or p in {'.DS_Store','__MACOSX','.AppleDouble','__pycache__','.pytest_cache','.cache','.venv'}
or p.endswith(('.pyc','.tmp','.swp')) for p in Path(path).parts)
def assemble():
files=[]
def add(name,path):
if excluded(name) or excluded(path):
raise ValueError(f'forbidden release path: {name}')
if path.is_symlink() or not path.is_file():
raise ValueError(f'missing or symlinked file: {path}')
files.append((name,path))
runtime={p.name:sha256(p) for p in sorted((ROOT/'timestral_mlx').glob('*.py')) if not excluded(p)}
inventory=[]
for size in VARIANTS:
bundle=ROOT/'models'/size
manifest=json.loads((bundle/'manifest.json').read_text())
report=json.loads((ROOT/'reports'/f'{size}.json').read_text())
if not report['passed'] or len(report['fixtures'])!=7 or not all(r['passed'] for r in report['fixtures']):
raise ValueError(f'{size}: incomplete or failed validation')
if report['bundle_manifest_sha256']!=sha256(bundle/'manifest.json') or report['runtime_sha256']!=runtime:
raise ValueError(f'{size}: validation is stale')
for name,entry in manifest['files'].items():
if sha256(bundle/name)!=entry['sha256'] or (bundle/name).stat().st_size!=entry['bytes']:
raise ValueError(f'{size}: bundle hash/size mismatch for {name}')
source=manifest['source']
license_bytes=(bundle/'LICENSE').read_bytes()
if license_bytes!=(ROOT/'LICENSE-WEIGHTS').read_bytes():
raise ValueError('original variant licences differ; review required')
shutil.copyfile(ROOT/'models'/'source'/size/'README.md',bundle/'upstream-model-card.md')
if sha256(bundle/'upstream-model-card.md')!=source['files']['README.md']['sha256']:
raise ValueError('upstream model-card hash mismatch')
(bundle/'README.md').write_text(f'''# Timestral-{size} MLX FP32
Independent MLX conversion of [{source['repo_id']}](https://huggingface.co/{source['repo_id']})
at revision `{source['revision']}`. No training or quantization; original FP32 tensor
values verified unchanged after serialization. See [manifest.json](manifest.json).
Install the `timestral_mlx` package from the repository root, then:
```python
from timestral_mlx import TimestralForecaster
forecaster = TimestralForecaster.from_pretrained("./{size}")
forecast = forecaster.forecast(context, prediction_length=96)
```
The original Modified MIT weight licence and Mistral AI attribution are preserved
verbatim in [LICENSE](LICENSE). Runtime code is separately Apache-2.0.
See the [main model card](../README.md) and [parity report](../reports/{size}.json).
''')
for name in ('config.json','model.safetensors','LICENSE','manifest.json','README.md','upstream-model-card.md'):
add(f'{size}/{name}',bundle/name)
inventory.append(dict(variant=size,path=size,source=source['repo_id'],source_revision=source['revision'],
parameters=manifest['conversion']['parameter_count'],
weight_bytes=manifest['files']['model.safetensors']['bytes'],
manifest_sha256=sha256(bundle/'manifest.json'),validation=f'reports/{size}.json',passed=True))
full_context=json.loads((ROOT/'reports'/'full-context.json').read_text())
if {r['variant'] for r in full_context if r['passed']}!=set(VARIANTS):
raise ValueError('full-context smoke checks are incomplete')
installation=json.loads((ROOT/'reports'/'installation.json').read_text())
if not installation['offline'] or installation['torch_installed'] or {r['variant'] for r in installation['results'] if r['passed']}!=set(VARIANTS):
raise ValueError('clean offline wheel installation checks are incomplete')
if not json.loads((ROOT/'reports'/'tests.json').read_text())['passed']:
raise ValueError('test suite did not pass')
(ROOT/'models.json').write_text(json.dumps(dict(format='timestral-mlx-collection-v1',models=inventory),indent=2)+'\n')
for name in ('README.md','LICENSE','LICENSE-WEIGHTS','NOTICE','pyproject.toml','requirements.txt',
'requirements-validation.txt','.gitignore','MANIFEST.in','models.json','pins.json'):
add(name,ROOT/name)
for folder,pattern in (('timestral_mlx','*.py'),('scripts','*.py'),('examples','*.py'),('tests','*.py'),('reports','*.json'),('dist','*.whl')):
for path in sorted((ROOT/folder).glob(pattern)):
if not excluded(path): add(f'{folder}/{path.name}',path)
if not any(name.startswith('dist/') for name,_ in files):
raise ValueError('build the installable wheel before publishing')
(ROOT/'SHA256SUMS').write_text(''.join(f'{sha256(path)} {name}\n' for name,path in sorted(files)))
add('SHA256SUMS',ROOT/'SHA256SUMS')
return sorted(files)
if __name__=='__main__':
files=assemble()
print(json.dumps(dict(files=len(files),bytes=sum(p.stat().st_size for _,p in files)),indent=2))
|