# OWL-ViT: open-vocabulary object detection with Transformers

This page is the Transformers contract of the OWL-ViT MIC and also a usage example. [doc-builder](https://github.com/huggingface/doc-builder/blob/main/docs/runnable-code-blocks.md) renders the `runnable` blocks below as documentation, and its pytest plugin runs them as written against `google/owlvit-base-patch32`. The MIC runner runs the same blocks against each slim fixture and each checkpoint in [`integration.json`](../../integration.json). Lines marked `# doc-builder: hide` only matter there: they swap in the pinned checkpoint or variant and print the result record.

## Detect objects described by text

```py runnable:detect
import torch
from transformers import AutoProcessor, OwlViTForObjectDetection, set_seed
import json  # doc-builder: hide
import os  # doc-builder: hide

model_id = "google/owlvit-base-patch32"
model_id = os.environ.get("MIC_MODEL", model_id)  # doc-builder: hide
variant = json.loads(os.environ.get("MIC_MODEL_KWARGS", "{}"))  # doc-builder: hide
device = "cuda" if torch.cuda.is_available() else "cpu"
set_seed(0)  # seeds torch, NumPy, and Python: loading, dropout, and data augmentation draw random numbers

processor = AutoProcessor.from_pretrained(model_id)
model = OwlViTForObjectDetection.from_pretrained(model_id).to(device)
if variant:  # doc-builder: hide
    del model  # doc-builder: hide
    model = OwlViTForObjectDetection.from_pretrained(model_id, **variant).to(device)  # doc-builder: hide
model.eval()

# Two synthetic images and two text queries per image: a real batch makes padding and batching meaningful.
texts = [["a photo of a cat", "a photo of a dog"], ["a red car", "a blue car"]]
images = [torch.rand(3, 768, 768), torch.rand(3, 768, 768)]
# OWL-ViT's processor concatenates the per-image query encodings, so every image must carry the
# same number of queries; padding=True then pads the queries to a common token length.
# OWL-ViT pads each image's queries on its own: pad to max_length so they concatenate.
inputs = processor(text=texts, images=images, return_tensors="pt", padding="max_length")
inputs = inputs.to(device, dtype=model.dtype)

with torch.no_grad():
    outputs = model(**inputs)

# Keep only the top few detections per image, never every post-processed box.
target_sizes = torch.tensor([(768, 768), (768, 768)], device=device)
results = processor.post_process_grounded_object_detection(
    outputs=outputs, target_sizes=target_sizes, threshold=0.0, text_labels=texts
)
detections = {
    "boxes": [r["boxes"][:3].float().cpu().tolist() for r in results],
    "scores": [r["scores"][:3].float().cpu().tolist() for r in results],
    "labels": [r["text_labels"][:3] for r in results],
}
print(results[0]["boxes"][:3])
```

## One training step

```py runnable:detect:2
# OWL-ViT's public forward returns no loss, so build a small detection loss from its outputs:
# match each target box to the nearest predicted box and use class BCE plus box L1.
labels = [
    {
        "class_labels": torch.tensor([0, 1], device=device, dtype=torch.int64),
        "boxes": torch.tensor([[0.5, 0.5, 0.2, 0.2], [0.3, 0.3, 0.1, 0.1]], device=device, dtype=torch.float32),
    },
    {
        "class_labels": torch.tensor([0], device=device, dtype=torch.int64),
        "boxes": torch.tensor([[0.6, 0.6, 0.3, 0.3]], device=device, dtype=torch.float32),
    },
]

model.train()
optimizer = torch.optim.SGD([p for p in model.parameters() if p.requires_grad], lr=0.01)
with torch.autocast(device_type=model.device.type, dtype=model.dtype, enabled=model.dtype != torch.float32):
    outputs = model(**inputs)
    pred_boxes = outputs.pred_boxes  # (batch, num_queries, 4), normalized cx, cy, w, h
    pred_logits = outputs.logits  # (batch, num_queries, num_queries_text)
    loss = 0.0
    for i, target in enumerate(labels):
        boxes = target["boxes"]
        classes = target["class_labels"]
        # Nearest predicted box for each target box.
        dist = torch.cdist(boxes, pred_boxes[i])
        idx = dist.argmin(dim=1)
        loss = loss + torch.nn.functional.l1_loss(pred_boxes[i][idx], boxes)
        # Score every text query of the matched boxes: the zero targets keep a gradient
        # even where the fixture's logits for the wanted class are saturated.
        wanted = torch.nn.functional.one_hot(classes, pred_logits.shape[-1]).float()
        loss = loss + torch.nn.functional.binary_cross_entropy_with_logits(pred_logits[i][idx].float(), wanted)
    loss = loss / len(labels)
loss.backward()
before = {n: p.detach().cpu().clone() for n, p in model.named_parameters() if p.requires_grad}  # doc-builder: hide
optimizer.step()
print(f"loss: {loss.item():.4f}")

if os.environ.get("MIC_RUNNER"):  # doc-builder: hide
    named = dict(model.named_parameters())
    checked = ("class_head.dense0.weight", "owlvit.vision_model.encoder.layers.0.self_attn.q_proj.weight")
    result = {
        "runtime": {
            "device": model.device.type,
            "dtype": str(model.dtype).removeprefix("torch."),
            "attn_implementation": model.config._attn_implementation,
        },
        "detections": detections,
        "training": {
            "loss": loss.item(),
            "gradients": {n: bool(named[n].grad is not None and torch.count_nonzero(named[n].grad) > 0 and torch.isfinite(named[n].grad).all()) for n in checked},
            "updated": any(not torch.equal(p.detach().cpu(), before[n]) for n, p in model.named_parameters() if n in before),
        },
    }
    print(json.dumps(result))
```