File size: 4,037 Bytes
c83c617 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """Freeze a small code archive and explicit, checksummed AutoDL payload list.
No SSH, uploads, credential reads, model execution or training. Existing server
files with matching SHA256 can be reused when a later upload is authorized.
"""
import argparse
import json
from pathlib import Path
import sys
import tarfile
import tomllib
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
def main():
from vimeml.posttraining.train import build_plan
from vimeml.posttraining.infer import TRACKS, read_snapshot
from vimeml.training.data import file_sha, write_json
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, default=ROOT / "outputs/handoff-v4-c1")
parser.add_argument("--config", type=Path, default=ROOT / "configs/posttrain-v4-c1.toml")
args = parser.parse_args()
config_path = args.config.resolve()
config = tomllib.loads(config_path.read_text(encoding="utf-8"))
plan = build_plan(config, verify_files=True)
code = sorted((ROOT / "src/vimeml").rglob("*.py"))
code += sorted((ROOT / "scripts/posttraining").glob("*.py"))
code += [ROOT / "scripts/posttraining/autodl_v4_c1.sh", config_path]
code += [ROOT / name for name in ("requirements.txt", "requirements-autodl.txt", "pyproject.toml", "README.md", "LICENSE")]
code += [ROOT / name for name in ("docs/zh-CN/posttraining-v4.md", "docs/zh-CN/data-v4-c1.md",
"docs/posttraining-v4.md", "docs/data-v4-c1.md")]
data = [ROOT / p for p in plan["required_files"]]
data += [ROOT / config["conversion_dir"] / name for name in ("manifest.json", "acceptance.json")]
data += [ROOT / config["token_dir"] / "manifest.json", ROOT / config["index_dir"] / "manifest.json",
ROOT / config["tokenizer_dir"] / "tokenizer.model", ROOT / config["initialization"]["checkpoint"]]
for track in TRACKS:
read_snapshot(track)
data.append(ROOT / f"outputs/evaluation-v4-20261015/inputs/{track}.json")
args.output.mkdir(parents=True, exist_ok=True)
code = sorted(set(code))
data = sorted(set(data))
archive = args.output / "v4-c1-code.tar.gz"
def clean(info):
info.uid = info.gid = info.mtime = 0
info.uname = info.gname = ""
info.mode = 0o755 if info.name.endswith(".sh") else 0o644
return info
with tarfile.open(archive, "w:gz") as bundle:
for path in code:
bundle.add(path, arcname=path.relative_to(ROOT).as_posix(), recursive=False, filter=clean)
payload = [{"path": p.relative_to(ROOT).as_posix(), "bytes": p.stat().st_size, "sha256": file_sha(p)} for p in data]
result = {"format": "vimeml_v4_c1_handoff_v1", "code_archive": {"name": archive.name, "bytes": archive.stat().st_size,
"sha256": file_sha(archive)}, "code_files": {p.relative_to(ROOT).as_posix(): file_sha(p) for p in code},
"payload_files": payload, "payload_bytes": sum(x["bytes"] for x in payload),
"new_conversion_binary_bytes": sum(x["bytes"] for x in payload if x["path"].startswith(config["conversion_dir"]) and x["path"].endswith(".bin")),
"source_step": 584000, "max_steps": plan["settings"]["max_steps"], "signatures": plan["signatures"],
"purpose": "Frozen upload inventory; reuse identical remote files. No raw corpus, annotation dictionaries, credentials or blind data.",
"training_started": False, "remote_operations": False}
write_json(args.output / "manifest.json", result)
write_json(args.output / "training-plan.json", plan)
print(json.dumps({"output": str(args.output), "archive_bytes": result["code_archive"]["bytes"],
"payload_files": len(payload), "payload_GiB": result["payload_bytes"] / 2**30,
"new_conversion_binary_MiB": result["new_conversion_binary_bytes"] / 2**20,
"max_steps": result["max_steps"]}, indent=2))
if __name__ == "__main__":
main()
|