experiment_comparison.py

"""
Experiment: DINOv2-CLIP Hybrid vs Pure CLIP

Compares 4 approaches on the pre-computed EuroSAT gallery:
1. Pure CLIP (baseline)
2. CLIP + SAR Adapter
3. CLIP + DINOv2 patch features (hybrid)
4. Full hybrid (CLIP + SAR adapter + DINOv2)

Metrics: same-modal and cross-modal Recall@K, latency.
"""

import sys, time, json, traceback
import torch, numpy as np
from pathlib import Path
from PIL import Image
from dataclasses import dataclass, asdict
from typing import List, Optional

sys.path.insert(0, str(Path(__file__).parent))

DATA_DIR = Path("data")
PROCESSED_DIR = DATA_DIR / "processed"


@dataclass
class Result:
    model: str
    same_r1: float
    same_r5: float
    same_r10: float
    cross_r1: float
    cross_r5: float
    cross_r10: float
    latency_ms: float
    n_queries: int


def load_data():
    embeddings = torch.load(PROCESSED_DIR / "gallery_embeddings.pt", weights_only=True)
    with open(PROCESSED_DIR / "gallery_metadata.json") as f:
        metadata = json.load(f)
    return embeddings.numpy().astype(np.float32), metadata


def split(metadata):
    """Stratified 30/70 split: 30% of each (modality, class) pair goes to queries."""
    groups = {}
    for e in metadata:
        key = (e["modality"], e["class"])
        groups.setdefault(key, []).append(e)

    queries, gallery = [], []
    for key, entries in groups.items():
        n = max(1, int(len(entries) * 0.3))
        queries.extend(entries[:n])
        gallery.extend(entries[n:])
    return queries, gallery


def recall_at_k(retrieved, query_mod, query_class, metadata, k, mode="same"):
    hits = 0
    for idx in retrieved[:k]:
        m = metadata[idx]
        same_class = m["class"] == query_class
        same_mod = m["modality"] == query_mod
        if mode == "same" and same_class and same_mod:
            hits += 1
        elif mode == "cross" and same_class and not same_mod:
            hits += 1
    return hits


def evaluate(queries, all_emb, metadata, gallery_entries, extractor_fn, label):
    import faiss

    gal_idx = [e["index"] for e in gallery_entries]
    gal_emb = all_emb[gal_idx]
    dim = gal_emb.shape[1]
    index = faiss.IndexFlatIP(dim)
    index.add(gal_emb)

    sr1, sr5, sr10 = [], [], []
    cr1, cr5, cr10 = [], [], []
    latencies = []

    for q in queries:
        q_path = Path(q["gallery_path"])
        if not q_path.exists():
            continue
        img = Image.open(q_path).convert("RGB")

        start = time.perf_counter()
        try:
            emb = extractor_fn(img, q["modality"])
        except Exception:
            continue
        elapsed = (time.perf_counter() - start) * 1000
        latencies.append(elapsed)

        q_np = emb.reshape(1, -1).astype(np.float32)
        _, ids = index.search(q_np, 10)
        retrieved = [gal_idx[i] for i in ids[0] if 0 <= i < len(gal_idx)]

        sr1.append(recall_at_k(retrieved, q["modality"], q["class"], metadata, 1, "same"))
        sr5.append(recall_at_k(retrieved, q["modality"], q["class"], metadata, 5, "same"))
        sr10.append(recall_at_k(retrieved, q["modality"], q["class"], metadata, 10, "same"))
        cr1.append(recall_at_k(retrieved, q["modality"], q["class"], metadata, 1, "cross"))
        cr5.append(recall_at_k(retrieved, q["modality"], q["class"], metadata, 5, "cross"))
        cr10.append(recall_at_k(retrieved, q["modality"], q["class"], metadata, 10, "cross"))

    n = max(len(sr1), 1)
    return Result(
        model=label,
        same_r1=np.mean(sr1) / 1.0,
        same_r5=np.mean(sr5) / 5.0,
        same_r10=np.mean(sr10) / 10.0,
        cross_r1=np.mean(cr1) / 1.0,
        cross_r5=np.mean(cr5) / 5.0,
        cross_r10=np.mean(cr10) / 10.0,
        latency_ms=np.mean(latencies) if latencies else 0,
        n_queries=n,
    )


def main():
    print("=" * 72)
    print("  EXPERIMENT: DINOv2-CLIP Hybrid vs Pure CLIP")
    print("=" * 72)

    all_emb, metadata = load_data()
    queries, gallery = split(metadata)
    print(f"Gallery: {len(gallery)} | Queries: {len(queries)} | Dim: {all_emb.shape[1]}")

    from transformers import CLIPProcessor, CLIPModel
    device = "cuda" if torch.cuda.is_available() else "cpu"
    processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14")
    clip_model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14").to(device)
    clip_model.eval()
    print(f"CLIP loaded on {device}")

    @torch.no_grad()
    def clip_extract(img, modality):
        inputs = processor(images=img, return_tensors="pt").to(device)
        out = clip_model.vision_model(**inputs)
        pooled = out.last_hidden_state[:, 0, :]
        feat = clip_model.visual_projection(pooled).squeeze(0)
        return torch.nn.functional.normalize(feat, dim=-1).cpu().numpy()

    results = []

    print("\n[1/4] Pure CLIP ...")
    r = evaluate(queries, all_emb, metadata, gallery, clip_extract, "CLIP ViT-L/14")
    results.append(r)
    print(f"  Same R@5={r.same_r5:.4f} Cross R@5={r.cross_r5:.4f} Latency={r.latency_ms:.0f}ms")

    print("[2/4] CLIP + SAR Adapter ...")
    from src.features.sar_adapter import SARAdapter
    adapter = SARAdapter().eval()

    def clip_sar_extract(img, modality):
        if modality == "sar":
            arr = np.array(img).astype(np.float32) / 255.0
            t = torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0)
            with torch.no_grad():
                adapted = adapter(t)
            img = Image.fromarray((adapted.squeeze(0).permute(1, 2, 0).numpy() * 255).clip(0, 255).astype(np.uint8))
        return clip_extract(img, modality)

    r = evaluate(queries, all_emb, metadata, gallery, clip_sar_extract, "CLIP + SAR Adapter")
    results.append(r)
    print(f"  Same R@5={r.same_r5:.4f} Cross R@5={r.cross_r5:.4f} Latency={r.latency_ms:.0f}ms")

    print("[3/4] CLIP + DINOv2 Hybrid ...")
    try:
        dinov2 = torch.hub.load("facebookresearch/dinov2", "dinov2_vits14", pretrained=True)
        dinov2.to(device).eval()
        has_dino = True
        dino_embed_dim = dinov2.embed_dim  # 384 for vits14
        print(f"  DINOv2-ViT-S/14 loaded (embed_dim={dino_embed_dim})")
    except Exception as e:
        has_dino = False
        print(f"  DINOv2 load failed: {e}")

    if has_dino:
        from torchvision import transforms
        dino_transform = transforms.Compose([
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
        ])

        # Projection to match CLIP dim (768) if needed
        dino_proj = None
        if dino_embed_dim != 768:
            dino_proj = torch.nn.Linear(dino_embed_dim, 768, bias=False).to(device).eval()
            with torch.no_grad():
                torch.nn.init.eye_(dino_proj.weight)  # identity init — preserves features

        @torch.no_grad()
        def clip_dino_extract(img, modality):
            clip_feat = clip_extract(img, modality)
            t = dino_transform(img).unsqueeze(0).to(device)
            patch_feat = dinov2(t).squeeze(0)
            if dino_proj is not None:
                patch_feat = dino_proj(patch_feat)
            patch_feat = torch.nn.functional.normalize(patch_feat, dim=-1).cpu().numpy()
            hybrid = 0.7 * clip_feat + 0.3 * patch_feat
            return hybrid / (np.linalg.norm(hybrid) + 1e-8)

        r = evaluate(queries, all_emb, metadata, gallery, clip_dino_extract, "DINOv2-CLIP Hybrid")
        results.append(r)
        print(f"  Same R@5={r.same_r5:.4f} Cross R@5={r.cross_r5:.4f} Latency={r.latency_ms:.0f}ms")
    else:
        r = evaluate(queries, all_emb, metadata, gallery, clip_extract, "CLIP (DINOv2 unavailable)")
        results.append(r)

    print("[4/4] Full Hybrid (CLIP + SAR + DINOv2) ...")
    if has_dino:
        def full_extract(img, modality):
            if modality == "sar":
                arr = np.array(img).astype(np.float32) / 255.0
                t = torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0)
                with torch.no_grad():
                    adapted = adapter(t)
                img = Image.fromarray((adapted.squeeze(0).permute(1, 2, 0).numpy() * 255).clip(0, 255).astype(np.uint8))
            return clip_dino_extract(img, modality)

        r = evaluate(queries, all_emb, metadata, gallery, full_extract, "Full Hybrid (CLIP+SAR+DINOv2)")
        results.append(r)
        print(f"  Same R@5={r.same_r5:.4f} Cross R@5={r.cross_r5:.4f} Latency={r.latency_ms:.0f}ms")
    else:
        r = evaluate(queries, all_emb, metadata, gallery, clip_sar_extract, "CLIP+SAR (DINOv2 unavailable)")
        results.append(r)

    print("\n" + "=" * 72)
    print("  RESULTS")
    print("=" * 72)
    hdr = f"{'Model':<35} {'S-R@1':>6} {'S-R@5':>6} {'S-R@10':>7} {'C-R@1':>6} {'C-R@5':>6} {'C-R@10':>7} {'ms':>6}"
    print(hdr)
    print("-" * 72)
    for r in results:
        print(f"{r.model:<35} {r.same_r1:>6.4f} {r.same_r5:>6.4f} {r.same_r10:>7.4f} {r.cross_r1:>6.4f} {r.cross_r5:>6.4f} {r.cross_r10:>7.4f} {r.latency_ms:>5.0f}")

    base_s5 = results[0].same_r5
    base_c5 = results[0].cross_r5
    print(f"\nDelta vs CLIP baseline (R@5):")
    for r in results[1:]:
        ds = r.same_r5 - base_s5
        dc = r.cross_r5 - base_c5
        print(f"  {r.model}: Same {'+' if ds >= 0 else ''}{ds:.4f}, Cross {'+' if dc >= 0 else ''}{dc:.4f}")

    out = PROCESSED_DIR / "experiment_results.json"
    with open(out, "w") as f:
        json.dump([asdict(r) for r in results], f, indent=2)
    print(f"\nSaved to {out}")


if __name__ == "__main__":
    try:
        main()
    except Exception:
        traceback.print_exc()