| """Package committed Git history, V4.2 inference weights, PyTorch fixtures and frozen evidence for Mac Core ML work.""" |
|
|
| import argparse |
| import json |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import zipfile |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(ROOT / "src")) |
|
|
| MODEL_DIR = "artifacts/models/tiny-ja-v42-e8k-d384-l7" |
| TOKENIZER_DIR = "artifacts/tokenizers/v4-a/ja-unigram-8192" |
| TRAINING = "outputs/training-v42-20261020" |
| SOURCE_CHECKPOINT = f"{TRAINING}/remote/checkpoints/best-lm.pt" |
| REFERENCE = "outputs/deployment/tiny-ja-v42-epoch20-fp32-input" |
| STEP = 1081942 |
| INPUTS = [ |
| f"{TOKENIZER_DIR}/tokenizer.model", |
| f"{TOKENIZER_DIR}/tokenizer.vocab", |
| f"{TOKENIZER_DIR}/manifest.json", |
| "artifacts/token-data/corpus-v4-8k/manifest.json", |
| "outputs/evaluation-v4-20261015/inputs", |
| "outputs/evaluation-v4-20261015/v21", |
| "outputs/evaluation-v42-20261020", |
| f"{TRAINING}/download-receipt.json", |
| f"{TRAINING}/remote/manifest.json", |
| f"{TRAINING}/remote/records/config.json", |
| f"{TRAINING}/remote/records/plan.json", |
| f"{TRAINING}/remote/records/environment.json", |
| f"{TRAINING}/remote/records/summary.json", |
| f"{TRAINING}/remote/records/full-validation.json", |
| f"{TRAINING}/remote/records/epoch-validation-history.json", |
| |
| "artifacts/deployment/tiny-ja-v2.1-extend5-bundle-v1", |
| "artifacts/deployment/tiny-ja-v2.1-extend5-fp32-v1", |
| "artifacts/deployment/tiny-ja-v2.1-extend5-int8-b32-v1", |
| ] |
|
|
|
|
| def git(*args): |
| return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip() |
|
|
|
|
| def write_json(path, value): |
| path.parent.mkdir(parents=True, exist_ok=True) |
| path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") |
|
|
|
|
| def prepare_weights(stage): |
| import numpy as np |
| import sentencepiece as spm |
| import torch |
| from vimeml.training.data import file_sha |
| from vimeml.training.infer_v4 import JapaneseLMV4 |
|
|
| torch.set_num_threads(4) |
| checkpoint = ROOT / SOURCE_CHECKPOINT |
| receipt = json.loads((ROOT / TRAINING / "download-receipt.json").read_text(encoding="utf-8")) |
| expected = next(r["sha256"] for r in receipt["files"] if r["path"] == "checkpoints/best-lm.pt") |
| digest = file_sha(checkpoint) |
| if receipt["status"] != "verified" or digest != expected: |
| raise ValueError("best-lm.pt differs from the verified download receipt.") |
| plan = json.loads((ROOT / "outputs/evaluation-v42-20261020/plan.json").read_text(encoding="utf-8")) |
| if plan["checkpoint_sha256"] != digest: |
| raise ValueError("Checkpoint differs from the frozen local evaluation.") |
| saved = torch.load(checkpoint, map_location="cpu", weights_only=True) |
| if saved["format"] != "vimeml_tiny_gpt_v4" or saved["step"] != STEP: |
| raise ValueError("Expected frozen V4.2 best-lm epoch 20.") |
| exported = {key: saved[key] for key in ("format", "architecture", "model", "model_config", |
| "step", "epoch", "signatures", "tokenizer_identity")} |
| exported["config"] = {key: saved["config"][key] for key in ("architecture", "token_dir")} |
| exported["deployment_only"] = True |
| exported["source_checkpoint_sha256"] = digest |
| destination = stage / "payload" / MODEL_DIR / "deployment.pt" |
| destination.parent.mkdir(parents=True, exist_ok=True) |
| torch.save(exported, destination) |
| reloaded = torch.load(destination, map_location="cpu", weights_only=True) |
| if reloaded["model"].keys() != saved["model"].keys() or any( |
| not torch.equal(value, reloaded["model"][name]) for name, value in saved["model"].items()): |
| raise ValueError("Inference export changed a model tensor.") |
| del saved |
|
|
| lm = JapaneseLMV4(destination, ROOT / TOKENIZER_DIR, "cpu") |
| model = lm.model |
| processor = spm.SentencePieceProcessor(model_file=str(ROOT / TOKENIZER_DIR / "tokenizer.model")) |
| text = "今日はとてもいい天気です。駅まで歩いて、この本を読んで、日本語を勉強します。" |
| content = processor.encode(text * 20, out_type=int) |
| bos, pad = processor.bos_id(), processor.pad_id() |
| cases = {"bos": [bos], "short": [bos, *content[:15]], "long": [bos, *content[:127]]} |
| cases["right_pad"] = cases["short"] + [pad] * (128 - len(cases["short"])) |
| cases["changed_future"] = cases["short"] + content[:128 - len(cases["short"])] |
| arrays = {} |
| with torch.inference_mode(): |
| for name, ids in cases.items(): |
| inputs = torch.tensor([ids], dtype=torch.int32) |
| arrays[f"{name}_input_ids"] = inputs.numpy() |
| arrays[f"{name}_logits"] = model(inputs.long()).numpy() |
| |
| cache, steps = model.new_cache(), [] |
| long_ids = torch.tensor([cases["long"]], dtype=torch.int32) |
| for position in range(long_ids.shape[1]): |
| steps.append(model.forward_cached(long_ids[:, position:position + 1], cache)) |
| arrays["long_kv_step_logits"] = torch.cat(steps, dim=1).numpy() |
| reference = stage / "payload" / REFERENCE |
| reference.mkdir(parents=True) |
| np.savez_compressed(reference / "logits.npz", **arrays) |
| write_json(reference / "reference.json", { |
| "format": "vimeml_v4_mac_preparation_reference_v1", |
| "checkpoint_step": STEP, |
| "checkpoint_sha256": digest, |
| "tokenizer_sha256": lm.metadata["tokenizer_sha256"], |
| "precision": "fp32", "device": "cpu", "torch_version": torch.__version__, |
| "cases": {name: {"length": len(ids), |
| "valid_prefix": len(cases["short"]) if name in {"right_pad", "changed_future"} else len(ids)} |
| for name, ids in cases.items()}, |
| "kv_steps": {"case": "long", "array": "long_kv_step_logits", |
| "method": "TinyGPTV4.forward_cached one token at a time from an empty cache"}, |
| "policy": "PyTorch preparation fixtures; not Core ML validation, production quality or device timing.", |
| }) |
| return { |
| "checkpoint_step": STEP, |
| "checkpoint_epoch": 20, |
| "selection": "best complete LM validation BPC, fixed before local IME evaluation", |
| "source_checkpoint": SOURCE_CHECKPOINT, |
| "source_checkpoint_sha256": digest, |
| "deployment_checkpoint": f"{MODEL_DIR}/deployment.pt", |
| "deployment_checkpoint_sha256": file_sha(destination), |
| "tokenizer": TOKENIZER_DIR, |
| "tokenizer_sha256": lm.metadata["tokenizer_sha256"], |
| "model_config": lm.metadata["model"], |
| "parameter_count": model.parameter_count(), |
| "optimizer_included": False, |
| } |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--output", type=Path, required=True) |
| args = parser.parse_args() |
| output = args.output.resolve() |
| if output.exists(): |
| parser.error("Output exists; choose a new handoff version.") |
| if git("status", "--porcelain", "--untracked-files=no"): |
| parser.error("Commit tracked source changes before creating the portable Git bundle.") |
| branch = git("branch", "--show-current") |
| if not branch: |
| parser.error("A named source branch is required.") |
| from vimeml.training.data import file_sha |
| from vimeml.posttraining.infer import TRACKS, read_snapshot |
|
|
| for track in TRACKS: |
| read_snapshot(track) |
| maintenance = ROOT / "outputs/maintenance" |
| maintenance.mkdir(parents=True, exist_ok=True) |
| with tempfile.TemporaryDirectory(prefix="mac-v42-package-", dir=maintenance) as temporary: |
| stage = Path(temporary) |
| metadata = prepare_weights(stage) |
| payload = stage / "payload" |
| for relative in INPUTS: |
| source = ROOT / relative |
| if not source.exists(): |
| raise FileNotFoundError(source) |
| files = [source] if source.is_file() else sorted(p for p in source.rglob("*") if p.is_file()) |
| for path in files: |
| if (path.is_symlink() or path.name.startswith(".env") |
| or path.name in {".netrc", "credentials"} or path.suffix == ".pyc"): |
| raise ValueError(f"Unexpected input: {path}") |
| target = payload / path.relative_to(ROOT) |
| target.parent.mkdir(parents=True, exist_ok=True) |
| shutil.copy2(path, target) |
| subprocess.run(["git", "bundle", "create", str(stage / "repository.bundle"), branch], cwd=ROOT, check=True) |
| subprocess.run(["git", "bundle", "verify", str(stage / "repository.bundle")], cwd=ROOT, check=True, |
| stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) |
| manifest = { |
| "format": "vimeml_windows_to_mac_v42", |
| "source_commit": git("rev-parse", "HEAD"), |
| "source_branch": branch, |
| "mac_branch": "claude/mac-v42-coreml", |
| "checkout_name": "VimeML-v42-coreml", |
| "handoff_record": "outputs/deployment/mac-v42-handoff.json", |
| "remote": git("remote", "get-url", "origin"), |
| "model": metadata, |
| "reference": REFERENCE, |
| "frozen_tracks": {k: {"cases": v[0], "sha256": v[1]} for k, v in TRACKS.items()}, |
| "blind_included": False, |
| "test_included": False, |
| "coreml_conversion_done": False, |
| "payload_files": [ |
| {"path": p.relative_to(payload).as_posix(), "bytes": p.stat().st_size, "sha256": file_sha(p)} |
| for p in sorted(payload.rglob("*")) if p.is_file() |
| ], |
| "verification": "Receipt/evaluation checkpoint hash; exact tensor equality; pinned track hashes; Git bundle verify; per-file SHA256; ZIP entry paths/sizes.", |
| } |
| write_json(stage / "handoff-manifest.json", manifest) |
| shutil.copy2(ROOT / "scripts/deployment/setup_mac.py", stage / "setup_mac.py") |
| shutil.copy2(ROOT / "docs/reference/mac-v42-coreml.md", stage / "README.md") |
| output.parent.mkdir(parents=True, exist_ok=True) |
| expected = {} |
| with zipfile.ZipFile(output, "x", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive: |
| for path in sorted(stage.rglob("*")): |
| if path.is_file(): |
| name = path.relative_to(stage).as_posix() |
| archive.write(path, name) |
| expected[name] = path.stat().st_size |
| with zipfile.ZipFile(output) as archive: |
| if {p.filename: p.file_size for p in archive.infolist()} != expected or archive.testzip() is not None: |
| raise ValueError("ZIP directory or CRC differs from packaged files.") |
| summary = {"zip": output.name, "bytes": output.stat().st_size, "sha256": file_sha(output), |
| "file_count": len(expected), "source_commit": manifest["source_commit"], "model": metadata} |
| write_json(output.with_suffix(".json"), summary) |
| print(json.dumps({k: summary[k] for k in ("zip", "bytes", "sha256", "file_count", "source_commit")}, |
| ensure_ascii=False)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
