Argus demo Space

.gitattributes

@@ -0,0 +1,41 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/bear.jpg filter=lfs diff=lfs merge=lfs -text
+examples/cat.jpg filter=lfs diff=lfs merge=lfs -text
+examples/skier.jpg filter=lfs diff=lfs merge=lfs -text
+examples/banana.jpg filter=lfs diff=lfs merge=lfs -text
+argus_banner.png filter=lfs diff=lfs merge=lfs -text
+argus_avatar.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,25 @@
+---
+title: Argus
+emoji: 👁
+colorFrom: gray
+colorTo: indigo
+sdk: gradio
+sdk_version: 5.0.0
+python_version: "3.12"
+app_file: app.py
+pinned: false
+license: other
+license_name: fair-research-license
+models:
+  - phanerozoic/argus
+---
+
+# Argus
+
+Multi-task perception from a single 86-million-parameter backbone. Upload an
+image and Argus produces a top-5 classification, an ADE20K segmentation map,
+a metric depth map, and (with a second image) dense keypoint correspondences,
+all from one forward pass through a frozen EUPE-ViT-B encoder.
+
+See the [model card](https://huggingface.co/phanerozoic/argus) for benchmark
+results, training details, and licensing.

app.py

@@ -0,0 +1,191 @@
+import importlib.util
+import time
+
+import gradio as gr
+import numpy as np
+import torch
+from huggingface_hub import hf_hub_download
+from PIL import Image, ImageDraw
+
+REPO_ID = "phanerozoic/argus"
+
+
+def _load_argus():
+    argus_py_path = hf_hub_download(repo_id=REPO_ID, filename="argus.py")
+    spec = importlib.util.spec_from_file_location("argus_module", argus_py_path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    return mod.Argus.from_pretrained(REPO_ID, device=device)
+
+
+print("Loading Argus...")
+MODEL = _load_argus()
+print(f"Argus ready on {MODEL.device}")
+
+
+def colorize_segmentation(seg_map: np.ndarray, num_classes: int = 150) -> Image.Image:
+    rng = np.random.RandomState(42)
+    palette = rng.randint(0, 255, size=(num_classes + 1, 3), dtype=np.uint8)
+    palette[0] = [0, 0, 0]
+    color = palette[seg_map.astype(np.int64) % (num_classes + 1)]
+    return Image.fromarray(color)
+
+
+def colorize_depth(depth_map: np.ndarray) -> Image.Image:
+    d = depth_map - depth_map.min()
+    if d.max() > 0:
+        d = d / d.max()
+    grey = (255 * (1 - d)).astype(np.uint8)
+    rgb = np.stack([grey, grey, grey], axis=-1)
+    return Image.fromarray(rgb)
+
+
+def make_overlay(image: Image.Image, mask: Image.Image, alpha: float = 0.55) -> Image.Image:
+    base = image.convert("RGB").resize(mask.size)
+    return Image.blend(base, mask.convert("RGB"), alpha=alpha)
+
+
+def perceive(image):
+    if image is None:
+        return None, None, None, ""
+
+    image = image.convert("RGB")
+    result = MODEL.perceive(image)
+
+    classification = {
+        c["class_name"].split(",")[0]: float(c["score"])
+        for c in result["classification"]
+    }
+
+    seg_color = colorize_segmentation(result["segmentation"])
+    seg_overlay = make_overlay(image, seg_color, alpha=0.55)
+
+    depth_img = colorize_depth(result["depth"])
+
+    t = result["timings_ms"]
+    d = result["depth"]
+    stats = (
+        f"**Inference**  classify {t['classify']:.0f} ms · segment {t['segment']:.0f} ms · "
+        f"depth {t['depth']:.0f} ms · total {t['total']:.0f} ms  \n"
+        f"**Depth range**  {d.min():.2f} – {d.max():.2f} m"
+    )
+    return classification, seg_overlay, depth_img, stats
+
+
+def correspondence(src_image, tgt_image, grid_size: int = 5):
+    if src_image is None or tgt_image is None:
+        return None, ""
+
+    src = src_image.convert("RGB")
+    tgt = tgt_image.convert("RGB")
+    sw, sh = src.size
+    tw, th = tgt.size
+
+    src_pts = []
+    for y in range(grid_size):
+        for x in range(grid_size):
+            src_pts.append([sw * (x + 0.5) / grid_size, sh * (y + 0.5) / grid_size])
+
+    t0 = time.time()
+    pred_pts = MODEL.correspond(src, tgt, src_pts, resolution=512)
+    elapsed_ms = (time.time() - t0) * 1000
+
+    annotated = tgt.copy()
+    draw = ImageDraw.Draw(annotated)
+    n = len(pred_pts)
+    radius = max(6, min(tw, th) // 80)
+    for i, (px, py) in enumerate(pred_pts):
+        hue = int(255 * i / max(n - 1, 1))
+        color = (255, hue, 50)
+        draw.ellipse(
+            (px - radius, py - radius, px + radius, py + radius),
+            fill=color,
+            outline="white",
+            width=2,
+        )
+
+    stats = f"**Correspondence**  matched {n} keypoints in {elapsed_ms:.0f} ms"
+    return annotated, stats
+
+
+HEADER = """
+# Argus
+Multi-task perception from a single 86-million-parameter backbone.
+[model card](https://huggingface.co/phanerozoic/argus)
+"""
+
+CORRESPONDENCE_NOTE = """
+## Correspondence
+Upload two images of related objects (the same scene from a different angle, the
+same category but different instance, etc). Argus extracts dense features from
+both, then for each point in a 5×5 grid on the source it finds the best-matching
+point in the target by cosine similarity.
+"""
+
+
+with gr.Blocks(title="Argus", theme=gr.themes.Soft()) as demo:
+    gr.Image(
+        "argus_banner.png",
+        show_label=False,
+        show_download_button=False,
+        container=False,
+        interactive=False,
+        height=240,
+    )
+    gr.Markdown(HEADER)
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            input_image = gr.Image(type="pil", label="Input image", height=380)
+            gr.Examples(
+                examples=[
+                    ["examples/bear.jpg"],
+                    ["examples/cat.jpg"],
+                    ["examples/aircraft.jpg"],
+                    ["examples/skier.jpg"],
+                    ["examples/banana.jpg"],
+                ],
+                inputs=[input_image],
+                label="Examples",
+            )
+        with gr.Column(scale=1):
+            classification = gr.Label(
+                label="Classification (kNN over ImageNet-1k)",
+                num_top_classes=5,
+            )
+
+    with gr.Row():
+        seg_output = gr.Image(label="Segmentation (ADE20K, 150 classes)", height=380)
+        depth_output = gr.Image(label="Depth (NYU-trained, meters)", height=380)
+
+    stats_md = gr.Markdown("")
+
+    input_image.change(
+        fn=perceive,
+        inputs=[input_image],
+        outputs=[classification, seg_output, depth_output, stats_md],
+    )
+
+    gr.Markdown("---")
+    gr.Markdown(CORRESPONDENCE_NOTE)
+
+    with gr.Row():
+        corr_src = gr.Image(type="pil", label="Source image", height=380)
+        corr_out = gr.Image(label="Target with predicted matches", height=380, interactive=False)
+
+    with gr.Row():
+        corr_tgt = gr.Image(type="pil", label="Target image", height=200)
+        corr_btn = gr.Button("Find correspondences", variant="primary")
+
+    corr_stats = gr.Markdown("")
+
+    corr_btn.click(
+        fn=correspondence,
+        inputs=[corr_src, corr_tgt],
+        outputs=[corr_out, corr_stats],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,6 @@
+torch>=2.0.0
+torchvision>=0.15.0
+Pillow>=9.0.0
+numpy>=1.20.0
+huggingface_hub>=0.25.0,<1.0
+gradio>=5.0.0