# Grounding DINO: zero-shot object detection with Transformers

This page is the Transformers contract of the Grounding DINO MIC and also a usage example. [doc-builder](https://github.com/huggingface/doc-builder/blob/main/docs/runnable-code-blocks.md) renders the `runnable` blocks below as documentation, and its pytest plugin runs them as written against `IDEA-Research/grounding-dino-tiny`. The MIC runner runs the same blocks against each slim fixture and each checkpoint in [`integration.json`](../../integration.json). Lines marked `# doc-builder: hide` only matter there: they swap in the pinned checkpoint or variant and print the result record.

## Detect objects from text prompts

```py runnable:grounding_dino
import torch
from PIL import Image
import numpy as np
from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection, set_seed
import json  # doc-builder: hide
import os  # doc-builder: hide

model_id = "IDEA-Research/grounding-dino-tiny"
model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0)

processor = AutoProcessor.from_pretrained(model_id)
model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)
if variant:  # doc-builder: hide
    del model  # doc-builder: hide
    model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide

# Two synthetic images with different text prompts (lowercase phrases, each ending with a dot)
rng = np.random.default_rng(0)
image1 = Image.fromarray(rng.integers(0, 255, (320, 480, 3), dtype=np.uint8))
image2 = Image.fromarray(rng.integers(0, 255, (240, 320, 3), dtype=np.uint8))
text1 = "a cat. a remote control."
text2 = "a person. a dog."

inputs = processor(images=[image1, image2], text=[text1, text2], return_tensors="pt", padding=True)
inputs = inputs.to(device, dtype=model.dtype)  # casts only floating tensors (pixel_values)

with torch.no_grad():
    outputs = model(**inputs)

# Compact probes for both images: first 3 queries, their first 4 token scores and boxes
class_scores = outputs.logits[:, :3, :4].float().tolist()
box_preds = outputs.pred_boxes[:, :3].float().tolist()

results = processor.post_process_grounded_object_detection(
    outputs, inputs.input_ids, threshold=0.3, text_threshold=0.25, target_sizes=[image.size[::-1] for image in (image1, image2)]
)
print(f"detections per image: {[len(result['boxes']) for result in results]}")
```

## One training step

```py runnable:grounding_dino:2
# Labels: class_labels index the phrases of each prompt; boxes are normalized (cx, cy, w, h)
labels = [
    {
        "class_labels": torch.tensor([0], dtype=torch.long, device=device),  # "a cat"
        "boxes": torch.tensor([[0.5, 0.5, 0.2, 0.3]], dtype=torch.float32, device=device),
    },
    {
        "class_labels": torch.tensor([1], dtype=torch.long, device=device),  # "a dog"
        "boxes": torch.tensor([[0.6, 0.6, 0.3, 0.3]], dtype=torch.float32, device=device),
    },
]

model.train()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
# The Hungarian matcher uses torch.cdist, which has no bfloat16 kernel: autocast runs it in float32
with torch.autocast(device, dtype=model.dtype, enabled=model.dtype != torch.float32):
    loss = model(**inputs, labels=labels).loss
loss.backward()
before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")

if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
    named = dict(model.named_parameters())
    checked = ("model.text_backbone.embeddings.word_embeddings.weight", "model.decoder.layers.0.self_attn.query.weight")
    result = {
        "runtime": {
            "device": model.device.type,
            "dtype": str(model.dtype).removeprefix("torch."),
            "attn_implementation": model.config._attn_implementation,
        },
        "class_scores": class_scores,
        "box_preds": box_preds,
        "training": {
            "loss": loss.item(),
            "gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked},
            "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),
        },
    }
    print(json.dumps(result))
```