Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / transformers /pr_33892 /en /tasks /zero_shot_object_detection.html

rtrm

about 1 month ago

download

raw

58.4 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Zero-shot object detection","local":"zero-shot-object-detection","sections":[{"title":"Zero-shot object detection pipeline","local":"zero-shot-object-detection-pipeline","sections":[],"depth":2},{"title":"Text-prompted zero-shot object detection by hand","local":"text-prompted-zero-shot-object-detection-by-hand","sections":[],"depth":2},{"title":"Batch processing","local":"batch-processing","sections":[],"depth":2},{"title":"Image-guided object detection","local":"image-guided-object-detection","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/transformers/pr_33892/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/entry/start.b2c4257a.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/scheduler.31fdf58d.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/singletons.9860629f.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/index.252883d5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/paths.e85c0ec8.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/entry/app.05ef1f97.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/preload-helper.40847a0e.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/index.2f76fdf0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/nodes/0.ca4aafa4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/nodes/582.63dee309.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/CopyLLMTxtMenu.ff482081.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.71f274cc.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/IconCopy.ac192424.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/CodeBlock.ab12f8e1.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/DocNotebookDropdown.dd28433e.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Zero-shot object detection","local":"zero-shot-object-detection","sections":[{"title":"Zero-shot object detection pipeline","local":"zero-shot-object-detection-pipeline","sections":[],"depth":2},{"title":"Text-prompted zero-shot object detection by hand","local":"text-prompted-zero-shot-object-detection-by-hand","sections":[],"depth":2},{"title":"Batch processing","local":"batch-processing","sections":[],"depth":2},{"title":"Image-guided object detection","local":"image-guided-object-detection","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 max-sm:gap-0.5 h-6 max-sm:h-5 px-2 max-sm:px-1.5 text-[11px] max-sm:text-[9px] font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0"><svg class="w-3 h-3 max-sm:w-2.5 max-sm:h-2.5" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-6 max-sm:h-5 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible w-3 h-3 max-sm:w-2.5 max-sm:h-2.5 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <div class="flex space-x-1 " style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"> <div class="relative colab-dropdown "> <button class=" " type="button"> <img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"> </button> </div> <div class="relative colab-dropdown "> <button class=" " type="button"> <img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"> </button> </div></div> <h1 class="relative group"><a id="zero-shot-object-detection" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#zero-shot-object-detection"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Zero-shot object detection</span></h1> <p data-svelte-h="svelte-1g13my9">Traditionally, models used for <a href="object_detection">object detection</a> require labeled image datasets for training,
	and are limited to detecting the set of classes from the training data.</p> <p data-svelte-h="svelte-18e0bwc">Zero-shot object detection is a computer vision task to detect objects and their classes in images, without any
	prior training or knowledge of the classes. Zero-shot object detection models receive an image as input, as well
	as a list of candidate classes, and output the bounding boxes and labels where the objects have been detected.</p> <blockquote class="note" data-svelte-h="svelte-ktqmmu"><p>Hugging Face houses many such <a href="https://huggingface.co/models?pipeline_tag=zero-shot-object-detection" rel="nofollow">open vocabulary zero shot object detectors</a>.</p></blockquote> <p data-svelte-h="svelte-9bp1kt">In this guide, you will learn how to use such models:</p> <ul data-svelte-h="svelte-v8ep8v"><li>to detect objects based on text prompts</li> <li>for batch object detection</li> <li>for image-guided object detection</li></ul> <p data-svelte-h="svelte-1c9nexd">Before you begin, make sure you have all the necessary libraries installed:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install -q transformers<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="zero-shot-object-detection-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#zero-shot-object-detection-pipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Zero-shot object detection pipeline</span></h2> <p data-svelte-h="svelte-rtnnp7">The simplest way to try out inference with models is to use it in a <a href="/docs/transformers/pr_33892/en/main_classes/pipelines#transformers.pipeline">pipeline()</a>. Instantiate a pipeline
	for zero-shot object detection from a <a href="https://huggingface.co/models?pipeline_tag=zero-shot-object-detection" rel="nofollow">checkpoint on the Hugging Face Hub</a>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

	<span class="hljs-meta">>>> </span><span class="hljs-comment"># Use any checkpoint from the hf.co/models?pipeline_tag=zero-shot-object-detection</span>
	<span class="hljs-meta">>>> </span>checkpoint = <span class="hljs-string">"iSEE-Laboratory/llmdet_large"</span>
	<span class="hljs-meta">>>> </span>detector = pipeline(model=checkpoint, task=<span class="hljs-string">"zero-shot-object-detection"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-henpll">Next, choose an image you’d like to detect objects in. Here we’ll use the image of astronaut Eileen Collins that is
	a part of the <a href="https://www.nasa.gov/multimedia/imagegallery/index.html" rel="nofollow">NASA</a> Great Images dataset.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers.image_utils <span class="hljs-keyword">import</span> load_image

	<span class="hljs-meta">>>> </span>url = <span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_1.png"</span>
	<span class="hljs-meta">>>> </span>image = load_image(url)
	<span class="hljs-meta">>>> </span>image<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-17qmfee"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_1.png" alt="Astronaut Eileen Collins"></div> <p data-svelte-h="svelte-1sgtlni">Pass the image and the candidate object labels to look for to the pipeline.
	Here we pass the image directly; other suitable options include a local path to an image or an image url. We also pass text descriptions for all items we want to query the image for.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>predictions = detector(
	<span class="hljs-meta">... </span> image,
	<span class="hljs-meta">... </span> candidate_labels=[<span class="hljs-string">"human face"</span>, <span class="hljs-string">"rocket"</span>, <span class="hljs-string">"nasa badge"</span>, <span class="hljs-string">"star-spangled banner"</span>],
	<span class="hljs-meta">... </span> threshold=<span class="hljs-number">0.45</span>,
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>predictions
	[{<span class="hljs-string">'score'</span>: <span class="hljs-number">0.8409242033958435</span>,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'human face'</span>,
	<span class="hljs-string">'box'</span>: {<span class="hljs-string">'xmin'</span>: <span class="hljs-number">179</span>, <span class="hljs-string">'ymin'</span>: <span class="hljs-number">74</span>, <span class="hljs-string">'xmax'</span>: <span class="hljs-number">272</span>, <span class="hljs-string">'ymax'</span>: <span class="hljs-number">179</span>}},
	{<span class="hljs-string">'score'</span>: <span class="hljs-number">0.7380027770996094</span>,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'rocket'</span>,
	<span class="hljs-string">'box'</span>: {<span class="hljs-string">'xmin'</span>: <span class="hljs-number">353</span>, <span class="hljs-string">'ymin'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'xmax'</span>: <span class="hljs-number">466</span>, <span class="hljs-string">'ymax'</span>: <span class="hljs-number">284</span>}},
	{<span class="hljs-string">'score'</span>: <span class="hljs-number">0.5850900411605835</span>,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'star-spangled banner'</span>,
	<span class="hljs-string">'box'</span>: {<span class="hljs-string">'xmin'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'ymin'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'xmax'</span>: <span class="hljs-number">96</span>, <span class="hljs-string">'ymax'</span>: <span class="hljs-number">511</span>}},
	{<span class="hljs-string">'score'</span>: <span class="hljs-number">0.5697067975997925</span>,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'human face'</span>,
	<span class="hljs-string">'box'</span>: {<span class="hljs-string">'xmin'</span>: <span class="hljs-number">18</span>, <span class="hljs-string">'ymin'</span>: <span class="hljs-number">15</span>, <span class="hljs-string">'xmax'</span>: <span class="hljs-number">366</span>, <span class="hljs-string">'ymax'</span>: <span class="hljs-number">511</span>}},
	{<span class="hljs-string">'score'</span>: <span class="hljs-number">0.47813931107521057</span>,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'star-spangled banner'</span>,
	<span class="hljs-string">'box'</span>: {<span class="hljs-string">'xmin'</span>: <span class="hljs-number">353</span>, <span class="hljs-string">'ymin'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'xmax'</span>: <span class="hljs-number">459</span>, <span class="hljs-string">'ymax'</span>: <span class="hljs-number">274</span>}},
	{<span class="hljs-string">'score'</span>: <span class="hljs-number">0.46597740054130554</span>,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'nasa badge'</span>,
	<span class="hljs-string">'box'</span>: {<span class="hljs-string">'xmin'</span>: <span class="hljs-number">353</span>, <span class="hljs-string">'ymin'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'xmax'</span>: <span class="hljs-number">462</span>, <span class="hljs-string">'ymax'</span>: <span class="hljs-number">279</span>}},
	{<span class="hljs-string">'score'</span>: <span class="hljs-number">0.4585932493209839</span>,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'nasa badge'</span>,
	<span class="hljs-string">'box'</span>: {<span class="hljs-string">'xmin'</span>: <span class="hljs-number">132</span>, <span class="hljs-string">'ymin'</span>: <span class="hljs-number">348</span>, <span class="hljs-string">'xmax'</span>: <span class="hljs-number">208</span>, <span class="hljs-string">'ymax'</span>: <span class="hljs-number">423</span>}}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-af5rkc">Let’s visualize the predictions:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> ImageDraw

	<span class="hljs-meta">>>> </span>draw = ImageDraw.Draw(image)

	<span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> prediction <span class="hljs-keyword">in</span> predictions:
	<span class="hljs-meta">... </span> box = prediction[<span class="hljs-string">"box"</span>]
	<span class="hljs-meta">... </span> label = prediction[<span class="hljs-string">"label"</span>]
	<span class="hljs-meta">... </span> score = prediction[<span class="hljs-string">"score"</span>]

	<span class="hljs-meta">... </span> xmin, ymin, xmax, ymax = box.values()
	<span class="hljs-meta">... </span> draw.rectangle((xmin, ymin, xmax, ymax), outline=<span class="hljs-string">"red"</span>, width=<span class="hljs-number">1</span>)
	<span class="hljs-meta">... </span> draw.text((xmin, ymin), <span class="hljs-string">f"<span class="hljs-subst">{label}</span>: <span class="hljs-subst">{<span class="hljs-built_in">round</span>(score,<span class="hljs-number">2</span>)}</span>"</span>, fill=<span class="hljs-string">"white"</span>)

	<span class="hljs-meta">>>> </span>image<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-1fwpqdn"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_2.png" alt="Visualized predictions on NASA image"></div> <h2 class="relative group"><a id="text-prompted-zero-shot-object-detection-by-hand" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-prompted-zero-shot-object-detection-by-hand"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-prompted zero-shot object detection by hand</span></h2> <p data-svelte-h="svelte-ul7h2w">Now that you’ve seen how to use the zero-shot object detection pipeline, let’s replicate the same result manually.</p> <p data-svelte-h="svelte-4n9h0j">Start by loading the model and associated processor from a <a href="hf.co/iSEE-Laboratory/llmdet_large">checkpoint on the Hugging Face Hub</a>.
	Here we’ll use the same checkpoint as before:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, AutoModelForZeroShotObjectDetection

	<span class="hljs-meta">>>> </span>model = AutoModelForZeroShotObjectDetection.from_pretrained(checkpoint, device_map=<span class="hljs-string">"auto"</span>)
	<span class="hljs-meta">>>> </span>processor = AutoProcessor.from_pretrained(checkpoint)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1g7c1zc">Let’s take a different image to switch things up.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>url = <span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_3.png"</span>
	<span class="hljs-meta">>>> </span>image = load_image(url)
	<span class="hljs-meta">>>> </span>image<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-owux8y"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_3.png" alt="Beach photo"></div> <p data-svelte-h="svelte-i7d56l">Use the processor to prepare the inputs for the model.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>text_labels = [<span class="hljs-string">"hat"</span>, <span class="hljs-string">"book"</span>, <span class="hljs-string">"sunglasses"</span>, <span class="hljs-string">"camera"</span>]
	<span class="hljs-meta">>>> </span>inputs = processor(text=text_labels, images=image, return_tensors=<span class="hljs-string">"pt"</span>)to(model.device)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-f7ur0a">Pass the inputs through the model, post-process, and visualize the results. Since the image processor resized images before
	feeding them to the model, you need to use the <code>post_process_object_detection</code> method to make sure the predicted bounding
	boxes have the correct coordinates relative to the original image:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch

	<span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.inference_mode():
	<span class="hljs-meta">... </span> outputs = model(**inputs)

	<span class="hljs-meta">>>> </span>results = processor.post_process_grounded_object_detection(
	<span class="hljs-meta">... </span> outputs, threshold=<span class="hljs-number">0.50</span>, target_sizes=[(image.height, image.width)], text_labels=text_labels,
	...)[<span class="hljs-number">0</span>]

	<span class="hljs-meta">>>> </span>draw = ImageDraw.Draw(image)

	<span class="hljs-meta">>>> </span>scores = results[<span class="hljs-string">"scores"</span>]
	<span class="hljs-meta">>>> </span>text_labels = results[<span class="hljs-string">"text_labels"</span>]
	<span class="hljs-meta">>>> </span>boxes = results[<span class="hljs-string">"boxes"</span>]

	<span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> box, score, text_label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(boxes, scores, text_labels):
	<span class="hljs-meta">... </span> xmin, ymin, xmax, ymax = box
	<span class="hljs-meta">... </span> draw.rectangle((xmin, ymin, xmax, ymax), outline=<span class="hljs-string">"red"</span>, width=<span class="hljs-number">1</span>)
	<span class="hljs-meta">... </span> draw.text((xmin, ymin), <span class="hljs-string">f"<span class="hljs-subst">{text_label}</span>: <span class="hljs-subst">{<span class="hljs-built_in">round</span>(score.item(),<span class="hljs-number">2</span>)}</span>"</span>, fill=<span class="hljs-string">"white"</span>)

	<span class="hljs-meta">>>> </span>image<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-1m863ar"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_4.png" alt="Beach photo with detected objects"></div> <h2 class="relative group"><a id="batch-processing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#batch-processing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Batch processing</span></h2> <p data-svelte-h="svelte-16j89af">You can pass multiple sets of images and text queries to search for different (or same) objects in several images.
	Let’s use both an astronaut image and the beach image together.
	For batch processing, you should pass text queries as a nested list to the processor and images as lists of PIL images,
	PyTorch tensors, or NumPy arrays.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>url1 = <span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_1.png"</span>
	<span class="hljs-meta">>>> </span>url2 = <span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_3.png"</span>
	<span class="hljs-meta">>>> </span>images = [load_image(url1), load_image(url2)]
	<span class="hljs-meta">>>> </span>text_queries = [
	<span class="hljs-meta">... </span> [<span class="hljs-string">"human face"</span>, <span class="hljs-string">"rocket"</span>, <span class="hljs-string">"nasa badge"</span>, <span class="hljs-string">"star-spangled banner"</span>],
	<span class="hljs-meta">... </span> [<span class="hljs-string">"hat"</span>, <span class="hljs-string">"book"</span>, <span class="hljs-string">"sunglasses"</span>, <span class="hljs-string">"camera"</span>, <span class="hljs-string">"can"</span>],
	<span class="hljs-meta">... </span>]
	<span class="hljs-meta">>>> </span>inputs = processor(text=text_queries, images=images, return_tensors=<span class="hljs-string">"pt"</span>, padding=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1si811t">Previously for post-processing you passed the single image’s size as a tensor, but you can also pass a tuple, or, in case
	of several images, a list of tuples. Let’s create predictions for the two examples, and visualize the second one (<code>image_idx = 1</code>).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad():
	<span class="hljs-meta">>>> </span> outputs = model(**inputs)

	<span class="hljs-meta">>>> </span>target_sizes = [(image.height, image.width) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> images]
	<span class="hljs-meta">>>> </span>results = processor.post_process_grounded_object_detection(
	<span class="hljs-meta">... </span> outputs, threshold=<span class="hljs-number">0.3</span>, target_sizes=target_sizes, text_labels=text_labels,
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-19l8kfc">Let’s visualize the results:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>image_idx = <span class="hljs-number">1</span>
	<span class="hljs-meta">>>> </span>draw = ImageDraw.Draw(images[image_idx])

	<span class="hljs-meta">>>> </span>scores = results[image_idx][<span class="hljs-string">"scores"</span>].tolist()
	<span class="hljs-meta">>>> </span>text_labels = results[image_idx][<span class="hljs-string">"text_labels"</span>]
	<span class="hljs-meta">>>> </span>boxes = results[image_idx][<span class="hljs-string">"boxes"</span>].tolist()

	<span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> box, score, text_label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(boxes, scores, text_labels):
	<span class="hljs-meta">>>> </span> xmin, ymin, xmax, ymax = box
	<span class="hljs-meta">>>> </span> draw.rectangle((xmin, ymin, xmax, ymax), outline=<span class="hljs-string">"red"</span>, width=<span class="hljs-number">1</span>)
	<span class="hljs-meta">>>> </span> draw.text((xmin, ymin), <span class="hljs-string">f"<span class="hljs-subst">{text_label}</span>: <span class="hljs-subst">{<span class="hljs-built_in">round</span>(score,<span class="hljs-number">2</span>)}</span>"</span>, fill=<span class="hljs-string">"white"</span>)

	<span class="hljs-meta">>>> </span>images[image_idx]<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-1m863ar"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_4.png" alt="Beach photo with detected objects"></div> <h2 class="relative group"><a id="image-guided-object-detection" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-guided-object-detection"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Image-guided object detection</span></h2> <p data-svelte-h="svelte-dikxi5">In addition to zero-shot object detection with text queries, models like <a href="https://huggingface.co/collections/ariG23498/owlvit-689b0d0872a7634a6ea17ae7" rel="nofollow">OWL-ViT</a> and <a href="https://huggingface.co/collections/ariG23498/owlv2-689b0d27bd7d96ba3c7f7530" rel="nofollow">OWLv2</a> offers image-guided object detection. This means you can use an image query to find similar
	objects in the target image.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, AutoModelForZeroShotObjectDetection

	<span class="hljs-meta">>>> </span>checkpoint = <span class="hljs-string">"google/owlv2-base-patch16-ensemble"</span>
	<span class="hljs-meta">>>> </span>model = AutoModelForZeroShotObjectDetection.from_pretrained(checkpoint, device_map=<span class="hljs-string">"auto"</span>)
	<span class="hljs-meta">>>> </span>processor = AutoProcessor.from_pretrained(checkpoint)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-s3nt8l">Unlike text queries, only a single example image is allowed.</p> <p data-svelte-h="svelte-1kqxako">Let’s take an image with two cats on a couch as a target image, and an image of a single cat
	as a query:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>url = <span class="hljs-string">"http://images.cocodataset.org/val2017/000000039769.jpg"</span>
	<span class="hljs-meta">>>> </span>image_target = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

	<span class="hljs-meta">>>> </span>query_url = <span class="hljs-string">"http://images.cocodataset.org/val2017/000000524280.jpg"</span>
	<span class="hljs-meta">>>> </span>query_image = Image.<span class="hljs-built_in">open</span>(requests.get(query_url, stream=<span class="hljs-literal">True</span>).raw)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1yw5ubp">Let’s take a quick look at the images:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

	<span class="hljs-meta">>>> </span>fig, ax = plt.subplots(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>)
	<span class="hljs-meta">>>> </span>ax[<span class="hljs-number">0</span>].imshow(image_target)
	<span class="hljs-meta">>>> </span>ax[<span class="hljs-number">1</span>].imshow(query_image)
	<span class="hljs-meta">>>> </span>fig.show()<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-y78yu"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_5.png" alt="Cats"></div> <p data-svelte-h="svelte-34zysh">In the preprocessing step, instead of text queries, you now need to use <code>query_images</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>inputs = processor(images=image_target, query_images=query_image, return_tensors=<span class="hljs-string">"pt"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1yf37an">For predictions, instead of passing the inputs to the model, pass them to <a href="/docs/transformers/pr_33892/en/model_doc/owlvit#transformers.OwlViTForObjectDetection.image_guided_detection">image_guided_detection()</a>. Draw the predictions
	as before except now there are no labels.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad():
	<span class="hljs-meta">... </span> outputs = model.image_guided_detection(**inputs)
	<span class="hljs-meta">... </span> target_sizes = torch.tensor([image_target.size[::-<span class="hljs-number">1</span>]])
	<span class="hljs-meta">... </span> results = processor.post_process_image_guided_detection(outputs=outputs, target_sizes=target_sizes)[<span class="hljs-number">0</span>]

	<span class="hljs-meta">>>> </span>draw = ImageDraw.Draw(image_target)

	<span class="hljs-meta">>>> </span>scores = results[<span class="hljs-string">"scores"</span>].tolist()
	<span class="hljs-meta">>>> </span>boxes = results[<span class="hljs-string">"boxes"</span>].tolist()

	<span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> box, score <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(boxes, scores):
	<span class="hljs-meta">... </span> xmin, ymin, xmax, ymax = box
	<span class="hljs-meta">... </span> draw.rectangle((xmin, ymin, xmax, ymax), outline=<span class="hljs-string">"white"</span>, width=<span class="hljs-number">4</span>)

	<span class="hljs-meta">>>> </span>image_target<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-1f4dev0"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/zero-sh-obj-detection_6.png" alt="Cats with bounding boxes"></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/tasks/zero_shot_object_detection.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_16tnnm8 = {
	assets: "/docs/transformers/pr_33892/en",
	base: "/docs/transformers/pr_33892/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/transformers/pr_33892/en/_app/immutable/entry/start.b2c4257a.js"),
	import("/docs/transformers/pr_33892/en/_app/immutable/entry/app.05ef1f97.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 582],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 58.4 kB
Xet hash:: 9c62d1d3aecc122030853df197af06e210ae40369024eeec53744d93c0d0c25e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.