convert_inverse_renderer_1024.py

#!/usr/bin/env python3
"""Convert GilgameshYX InverseRenderer-1024 into BiliSakura IntrisicWeather-diffusers layout."""

from __future__ import annotations

import json
import shutil
import sys
from pathlib import Path

from diffusers.models.transformers import SD3Transformer2DModel

COLLECTION_ROOT = Path(__file__).resolve().parent
INTRINSIC_REPO = Path("/data/projects/IntrinsicWeather-diffusers")
sys.path.insert(0, str(INTRINSIC_REPO / "src"))
sys.path.insert(0, str(INTRINSIC_REPO))

from intrinsic_weather.models.transformers.transformer_intrinsic_weather import (  # noqa: E402
    IntrinsicWeatherSD3Transformer2DModel,
)
from scripts._conversion_utils import (  # noqa: E402
    ROOT as REPO_ROOT,
    expand_sd3_input_projection,
    merge_sharded_state_dict,
    save_imaa_bundle,
    write_scheduler_config,
)

SD3_PATH = Path(
    "/data/projects/Visual-Generative-Foundation-Model-Collection/models/stabilityai/stable-diffusion-3-medium-diffusers"
)
SD35_TRANSFORMER_REPO = "stabilityai/stable-diffusion-3.5-medium"
CKPT_PATH = Path(
    "/data/projects/Visual-Generative-Foundation-Model-Collection/models/GilgameshYX/InverseRenderer-1024"
)
OUTPUT_ROOT = COLLECTION_ROOT
TRANSFORMER_VARIANT = "inverse-1024"
SHARED_COMPONENTS = (
    "text_encoder",
    "text_encoder_2",
    "text_encoder_3",
    "tokenizer",
    "tokenizer_2",
    "tokenizer_3",
    "vae",
    "scheduler",
)


def copy_sd3_shared_components(sd3_path: Path, output_path: Path) -> None:
    for name in SHARED_COMPONENTS:
        src = sd3_path / name
        dst = output_path / name
        if dst.exists():
            print(f"Skipping existing shared component: {dst}")
            continue
        print(f"Copying {name} ...")
        shutil.copytree(src, dst)


def main() -> None:
    transformer_dir = OUTPUT_ROOT / "transformer" / TRANSFORMER_VARIANT
    transformer_dir.mkdir(parents=True, exist_ok=True)

    print(f"Ensuring shared SD3 components from {SD3_PATH} ...")
    copy_sd3_shared_components(SD3_PATH, OUTPUT_ROOT)
    write_scheduler_config(OUTPUT_ROOT)

    print("Converting inverse renderer transformer (1024) ...")
    base_transformer = SD3Transformer2DModel.from_config(
        SD3Transformer2DModel.load_config(SD35_TRANSFORMER_REPO, subfolder="transformer")
    )
    base_transformer = expand_sd3_input_projection(base_transformer, in_channels=32)
    custom_blocks = IntrinsicWeatherSD3Transformer2DModel.from_config(base_transformer.config)
    custom_blocks.load_state_dict(
        merge_sharded_state_dict(
            [
                CKPT_PATH / "pytorch_model-00001-of-00002.bin",
                CKPT_PATH / "pytorch_model-00002-of-00002.bin",
            ]
        ),
        strict=True,
    )
    custom_blocks.save_pretrained(transformer_dir.as_posix(), safe_serialization=True)
    shutil.copy2(
        REPO_ROOT / "src" / "intrinsic_weather" / "models" / "transformers" / "transformer_intrinsic_weather.py",
        transformer_dir / "transformer_intrinsic_weather.py",
    )

    print("Saving IMAA weights from InverseRenderer-1024 ...")
    save_imaa_bundle(CKPT_PATH / "imaa.pth", OUTPUT_ROOT, safe_serialization=True)

    conversion_metadata = {
        "task": "inverse_renderer",
        "resolution": 1024,
        "transformer_variant": TRANSFORMER_VARIANT,
        "source_transformer_checkpoints": [
            str((CKPT_PATH / "pytorch_model-00001-of-00002.bin").resolve()),
            str((CKPT_PATH / "pytorch_model-00002-of-00002.bin").resolve()),
        ],
        "source_imaa_checkpoint": str((CKPT_PATH / "imaa.pth").resolve()),
        "sd3_path": str(SD3_PATH.resolve()),
        "sd35_transformer_repo": SD35_TRANSFORMER_REPO,
        "in_channels": 32,
    }
    (OUTPUT_ROOT / "conversion_metadata_inverse_1024.json").write_text(
        json.dumps(conversion_metadata, indent=2) + "\n",
        encoding="utf-8",
    )
    print(f"Saved transformer to: {transformer_dir}")
    print("Load with: load_inverse_pipeline(transformer_subfolder='inverse-1024')")


if __name__ == "__main__":
    main()