src/scripts/prepare_hf_deployment.py

import shutil
from pathlib import Path
import sys

# Experiment directory to be used for deployment is passed in as input argument
exp_dir = Path(sys.argv[1])
deploy_dir = Path("hf_deployment")

# Safety-first: enables first seeing the changes before actually transfering files over (AWS s3 operations-like)
dry_run = "--dry-run" in sys.argv
checkpoint = deploy_dir / "checkpoint"
MODEL_SUBDIR = "model"
prefix = "[DRY RUN]" if dry_run else ""

print(f"{prefix} Create: {checkpoint}")
if not dry_run:
    checkpoint.mkdir(parents=True, exist_ok=True)
    (checkpoint / MODEL_SUBDIR).mkdir(exist_ok=True)

# Individual files
files = [
    (exp_dir / "config.json", checkpoint / "config.json"),
    (
        exp_dir / MODEL_SUBDIR / "pytorch_model.bin",
        checkpoint / MODEL_SUBDIR / "pytorch_model.bin",
    ),
]
for src, dst in files:
    print(f"{prefix} Copy: {src} -> {dst}")
    if not dry_run:
        shutil.copy2(src, dst)

# Directories (recursively)
trees = [
    (exp_dir / MODEL_SUBDIR / "tokenizer", checkpoint / MODEL_SUBDIR / "tokenizer"),
    (Path("src"), deploy_dir / "src"),
]
for src, dst in trees:
    print(f"{prefix} Copy tree: {src} -> {dst}")
    if not dry_run:
        shutil.copytree(src, dst, dirs_exist_ok=True)

if not dry_run:
    print(f"\nDeployment files are ready under {deploy_dir}.")