Grounding DINO: zero-shot object detection with Transformers

This page is the Transformers contract of the Grounding DINO MIC and also a usage example. doc-builder renders the runnable blocks below as documentation, and its pytest plugin runs them as written against IDEA-Research/grounding-dino-tiny. The MIC runner runs the same blocks against each slim fixture and each checkpoint in integration.json. Lines marked # doc-builder: hide only matter there: they swap in the pinned checkpoint or variant and print the result record.

Detect objects from text prompts

import torch
from PIL import Image
import numpy as np
from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection, set_seed
import json  # doc-builder: hide
import os  # doc-builder: hide

model_id = "IDEA-Research/grounding-dino-tiny"
model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0)

processor = AutoProcessor.from_pretrained(model_id)
model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)
if variant:  # doc-builder: hide
    del model  # doc-builder: hide
    model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide

# Two synthetic images with different text prompts (lowercase phrases, each ending with a dot)
rng = np.random.default_rng(0)
image1 = Image.fromarray(rng.integers(0, 255, (320, 480, 3), dtype=np.uint8))
image2 = Image.fromarray(rng.integers(0, 255, (240, 320, 3), dtype=np.uint8))
text1 = "a cat. a remote control."
text2 = "a person. a dog."

inputs = processor(images=[image1, image2], text=[text1, text2], return_tensors="pt", padding=True)
inputs = inputs.to(device, dtype=model.dtype)  # casts only floating tensors (pixel_values)

with torch.no_grad():
    outputs = model(**inputs)

# Compact probes for both images: first 3 queries, their first 4 token scores and boxes
class_scores = outputs.logits[:, :3, :4].float().tolist()
box_preds = outputs.pred_boxes[:, :3].float().tolist()

results = processor.post_process_grounded_object_detection(
    outputs, inputs.input_ids, threshold=0.3, text_threshold=0.25, target_sizes=[image.size[::-1] for image in (image1, image2)]
)
print(f"detections per image: {[len(result['boxes']) for result in results]}")

One training step

# Labels: class_labels index the phrases of each prompt; boxes are normalized (cx, cy, w, h)
labels = [
    {
        "class_labels": torch.tensor([0], dtype=torch.long, device=device),  # "a cat"
        "boxes": torch.tensor([[0.5, 0.5, 0.2, 0.3]], dtype=torch.float32, device=device),
    },
    {
        "class_labels": torch.tensor([1], dtype=torch.long, device=device),  # "a dog"
        "boxes": torch.tensor([[0.6, 0.6, 0.3, 0.3]], dtype=torch.float32, device=device),
    },
]

model.train()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
# The Hungarian matcher uses torch.cdist, which has no bfloat16 kernel: autocast runs it in float32
with torch.autocast(device, dtype=model.dtype, enabled=model.dtype != torch.float32):
    loss = model(**inputs, labels=labels).loss
loss.backward()
before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")

if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
    named = dict(model.named_parameters())
    checked = ("model.text_backbone.embeddings.word_embeddings.weight", "model.decoder.layers.0.self_attn.query.weight")
    result = {
        "runtime": {
            "device": model.device.type,
            "dtype": str(model.dtype).removeprefix("torch."),
            "attn_implementation": model.config._attn_implementation,
        },
        "class_scores": class_scores,
        "box_preds": box_preds,
        "training": {
            "loss": loss.item(),
            "gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked},
            "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),
        },
    }
    print(json.dumps(result))