examples/inference.py

import argparse
import json
from pathlib import Path
import sys

from PIL import Image
import torch
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file
from torchvision import transforms
from transformers import AutoConfig, AutoModel

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from projector import load_projector


MODEL_ID = "immanuelpeter/C-RADIOv4-H"
PATCH_SIZE = 16


def bundle_file(model: str, name: str) -> Path:
    path = Path(model)
    if path.is_dir():
        return path / name
    return Path(hf_hub_download(model, name))


def pixel_shuffle(tokens: torch.Tensor, scale_factor: float = 0.5) -> torch.Tensor:
    batch, width, height, channels = tokens.size()
    tokens = tokens.view(batch, width, int(height * scale_factor), int(channels / scale_factor))
    tokens = tokens.permute(0, 2, 1, 3).contiguous()
    tokens = tokens.view(
        batch,
        int(height * scale_factor),
        int(width * scale_factor),
        int(channels / (scale_factor * scale_factor)),
    )
    return tokens.permute(0, 2, 1, 3).contiguous()


def main() -> None:
    parser = argparse.ArgumentParser(description="Extract C-RADIOv4-H visual features.")
    parser.add_argument("image", type=Path)
    parser.add_argument("--model", default=MODEL_ID)
    args = parser.parse_args()

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    dtype = torch.bfloat16 if device.type == "cuda" else torch.float32
    config = json.loads(bundle_file(args.model, "config.json").read_text())
    source = AutoConfig.from_pretrained(config["source"], trust_remote_code=True, revision=config["revision"])
    radio = AutoModel.from_config(source.vision_config, trust_remote_code=True)
    generator = radio.radio_model.model.patch_generator
    if not hasattr(generator, "video_embedder"):
        generator.video_embedder = torch.nn.Linear(
            2 * 3 * generator.patch_size * generator.patch_size,
            generator.embed_dim,
            bias=False,
        )
    radio.load_state_dict(
        {name: tensor.to(dtype=dtype) for name, tensor in load_file(bundle_file(args.model, "model.safetensors")).items()},
        strict=False,
    )
    radio = radio.to(device=device, dtype=dtype)
    conditioner = radio.radio_model.input_conditioner
    if hasattr(conditioner, "dtype"):
        conditioner.dtype = dtype
    radio = radio.eval()
    projector = load_projector(args.model).to(device=device, dtype=dtype).eval()

    image = Image.open(args.image).convert("RGB")
    pixels = transforms.Compose([
        transforms.Resize(448, interpolation=transforms.InterpolationMode.BICUBIC),
        transforms.CenterCrop(448),
        transforms.ToTensor(),
    ])(image).unsqueeze(0).to(device=device, dtype=dtype)

    with torch.inference_mode():
        features = radio(pixels).features
        height = pixels.shape[-2] // PATCH_SIZE
        width = pixels.shape[-1] // PATCH_SIZE
        merged = pixel_shuffle(features.reshape(1, height, width, -1))
        merged = merged.reshape(1, -1, merged.shape[-1])
        projected = projector(merged)

    print("merged", merged.shape)
    print("projected", projected.shape)


if __name__ == "__main__":
    main()