infer.py

import os
import argparse

import cv2
import numpy as np
import torch
from safetensors.torch import load_file
from transformers import AutoImageProcessor

from lib.utils_segfly import ID2COLOR, mask2label
from lib.firefly_rgb import FireflyForSemanticSegmentationRGB, FireflyConfigRGB
from lib.firefly_thermal import FireflyForSemanticSegmentationThermal, FireflyConfigThermal


def run_infer(args):
    if not args.weights_path:
        if args.modality == "rgb":
            args.weights_path = "./Firefly_RGB/model.safetensors"
        else:
            args.weights_path = "./Firefly_Thermal/model.safetensors"
        print(f"No weights path provided, defaulting to: {args.weights_path}")

    weights_dir = os.path.dirname(args.weights_path) if os.path.isfile(args.weights_path) else args.weights_path

    print(f"Loading image processor from {weights_dir}...")
    image_processor = AutoImageProcessor.from_pretrained(weights_dir)

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    print(f"Initializing {args.modality} model...")
    if args.modality == "rgb":
        cfg = FireflyConfigRGB(
            num_labels=15,
            image_size=args.image_size,
            embedding_dim=256,
            backbone_embed_dim=768,
            patch_size=16,
            repo_dir=args.dinov3_repo_dir,
            model_name="dinov3_vitb16",
            semantic_loss_ignore_index=255,
        )
        model = FireflyForSemanticSegmentationRGB(cfg)
    else:
        cfg = FireflyConfigThermal(
            num_labels=15,
            image_size=args.image_size,
            embedding_dim=256,
            backbone_embed_dim=768,
            patch_size=16,
            num_layers=12,
            rein_token_length=100,
            feature_layers=[2, 5, 8, 11],
            repo_dir=args.dinov3_repo_dir,
            model_name="dinov3_vitb16",
            semantic_loss_ignore_index=255,
        )
        model = FireflyForSemanticSegmentationThermal(cfg)

    print(f"Loading weights from {args.weights_path}...")
    state_dict = load_file(args.weights_path)
    new_state_dict = {
        k[7:] if k.startswith("module.") else k: v
        for k, v in state_dict.items()
    }
    msg = model.load_state_dict(new_state_dict, strict=False)
    print(f"Weights loaded. Missing: {len(msg.missing_keys)}, Unexpected: {len(msg.unexpected_keys)}")

    model.eval()
    model.to(device)

    image_bgr = cv2.imread(args.image)
    if image_bgr is None:
        raise FileNotFoundError(f"Could not read image: {args.image}")
    image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
    orig_h, orig_w = image_rgb.shape[:2]

    inputs = image_processor(images=image_rgb, return_tensors="pt")
    pixel_values = inputs.pixel_values.to(device, dtype=torch.float32)

    with torch.no_grad():
        logits = model(pixel_values).logits
        upsampled = torch.nn.functional.interpolate(
            logits.float(), size=(orig_h, orig_w), mode="bilinear", align_corners=False
        )
        pred = upsampled.argmax(dim=1).squeeze(0).cpu().numpy()

    os.makedirs(args.output, exist_ok=True)

    pred_color = mask2label(pred, ID2COLOR)
    pred_bgr = cv2.cvtColor(pred_color, cv2.COLOR_RGB2BGR)

    stem = os.path.splitext(os.path.basename(args.image))[0]
    out_path = os.path.join(args.output, f"{stem}_pred.png")
    cv2.imwrite(out_path, pred_bgr)
    print(f"Segmentation saved to {out_path}")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Single-image inference for Firefly RGB / Thermal models")
    parser.add_argument("--image", type=str, required=True, help="Path to the input image.")
    parser.add_argument("--modality", type=str, default="rgb", choices=["rgb", "thermal"],
                        help="Model modality: 'rgb' or 'thermal'.")
    parser.add_argument("--output", type=str, default="./infer_output",
                        help="Directory to save the colorized segmentation output.")
    parser.add_argument("--weights_path", type=str, default="",
                        help="Path to model weights (.safetensors). Auto-detected from modality if not set.")
    parser.add_argument("--dinov3_repo_dir", type=str, default="./dinov3",
                        help="Path to the local DINOv3 repository.")
    parser.add_argument("--image_size", type=int, default=640,
                        help="Resolution at which the image is fed to the model.")
    args = parser.parse_args()
    run_infer(args)