Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / transformers /pr_33913 /en /tasks /semantic_segmentation.html

rtrm

about 1 month ago

download

raw

165 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Image Segmentation","local":"image-segmentation","sections":[{"title":"Types of Segmentation","local":"types-of-segmentation","sections":[],"depth":2},{"title":"Fine-tuning a Model for Segmentation","local":"fine-tuning-a-model-for-segmentation","sections":[{"title":"Load SceneParse150 dataset","local":"load-sceneparse150-dataset","sections":[{"title":"Custom dataset","local":"custom-dataset","sections":[],"depth":4}],"depth":3},{"title":"Preprocess","local":"preprocess","sections":[],"depth":3},{"title":"Evaluate","local":"evaluate","sections":[],"depth":3},{"title":"Train","local":"train","sections":[],"depth":3},{"title":"Inference","local":"inference","sections":[],"depth":3}],"depth":2}],"depth":1}">
	<link href="/docs/transformers/pr_33913/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/entry/start.b67f883f.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/scheduler.25b97de1.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/singletons.62a184e0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/index.e188933d.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/paths.51881b9e.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/entry/app.e436b1f2.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/index.d9030fc9.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/nodes/0.05e395f5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/nodes/434.63315461.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/Tip.baa67368.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/Youtube.eaf1a617.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/CodeBlock.e6cd0d95.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/DocNotebookDropdown.5ea6cb78.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/globals.7f7f1b26.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/Markdown.7217f838.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/stores.c3f24f16.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33913/en/_app/immutable/chunks/EditOnGithub.91d95064.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Image Segmentation","local":"image-segmentation","sections":[{"title":"Types of Segmentation","local":"types-of-segmentation","sections":[],"depth":2},{"title":"Fine-tuning a Model for Segmentation","local":"fine-tuning-a-model-for-segmentation","sections":[{"title":"Load SceneParse150 dataset","local":"load-sceneparse150-dataset","sections":[{"title":"Custom dataset","local":"custom-dataset","sections":[],"depth":4}],"depth":3},{"title":"Preprocess","local":"preprocess","sections":[],"depth":3},{"title":"Evaluate","local":"evaluate","sections":[],"depth":3},{"title":"Train","local":"train","sections":[],"depth":3},{"title":"Inference","local":"inference","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="image-segmentation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-segmentation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Image Segmentation</span></h1> <div class="flex space-x-1 absolute z-10 right-0 top-0"> <div class="relative colab-dropdown "> <button class=" " type="button"> <img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"> </button> </div> <div class="relative colab-dropdown "> <button class=" " type="button"> <img alt="Open In Studio Lab" class="!m-0" src="https://studiolab.sagemaker.aws/studiolab.svg"> </button> </div></div> <iframe class="w-full xl:w-4/6 h-80" src="https://www.youtube-nocookie.com/embed/dKE8SIt9C-w" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> <p data-svelte-h="svelte-i2v9l4">Image segmentation models separate areas corresponding to different areas of interest in an image. These models work by assigning a label to each pixel. There are several types of segmentation: semantic segmentation, instance segmentation, and panoptic segmentation.</p> <p data-svelte-h="svelte-1ad2x5z">In this guide, we will:</p> <ol data-svelte-h="svelte-phis9l"><li><a href="#types-of-segmentation">Take a look at different types of segmentation</a>.</li> <li><a href="#fine-tuning-a-model-for-segmentation">Have an end-to-end fine-tuning example for semantic segmentation</a>.</li></ol> <p data-svelte-h="svelte-1c9nexd">Before you begin, make sure you have all the necessary libraries installed:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># uncomment to install the necessary libraries</span>
	!pip install -q datasets transformers evaluate accelerate<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-27hn0u">We encourage you to log in to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to log in:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

	<span class="hljs-meta">>>> </span>notebook_login()<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="types-of-segmentation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#types-of-segmentation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Types of Segmentation</span></h2> <p data-svelte-h="svelte-1uhznwp">Semantic segmentation assigns a label or class to every single pixel in an image. Let’s take a look at a semantic segmentation model output. It will assign the same class to every instance of an object it comes across in an image, for example, all cats will be labeled as “cat” instead of “cat-1”, “cat-2”.
	We can use transformers’ image segmentation pipeline to quickly infer a semantic segmentation model. Let’s take a look at the example image.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
	<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
	<span class="hljs-keyword">import</span> requests

	url = <span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/segmentation_input.jpg"</span>
	image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)
	image<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-xdljqs"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/segmentation_input.jpg" alt="Segmentation Input"></div> <p data-svelte-h="svelte-1auudos">We will use <a href="https://huggingface.co/nvidia/segformer-b1-finetuned-cityscapes-1024-1024" rel="nofollow">nvidia/segformer-b1-finetuned-cityscapes-1024-1024</a>.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->semantic_segmentation = pipeline(<span class="hljs-string">"image-segmentation"</span>, <span class="hljs-string">"nvidia/segformer-b1-finetuned-cityscapes-1024-1024"</span>)
	results = semantic_segmentation(image)
	results<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-qumeg4">The segmentation pipeline output includes a mask for every predicted class.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'road'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'sidewalk'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'building'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'wall'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'pole'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'traffic sign'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'vegetation'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'terrain'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'sky'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: None,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'car'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-55zn24">Taking a look at the mask for the car class, we can see every car is classified with the same mask.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->results[-<span class="hljs-number">1</span>][<span class="hljs-string">"mask"</span>]<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-v81gtv"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/semantic_segmentation_output.png" alt="Semantic Segmentation Output"></div> <p data-svelte-h="svelte-11nij9p">In instance segmentation, the goal is not to classify every pixel, but to predict a mask for <strong>every instance of an object</strong> in a given image. It works very similar to object detection, where there is a bounding box for every instance, there’s a segmentation mask instead. We will use <a href="https://huggingface.co/facebook/mask2former-swin-large-cityscapes-instance" rel="nofollow">facebook/mask2former-swin-large-cityscapes-instance</a> for this.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->instance_segmentation = pipeline(<span class="hljs-string">"image-segmentation"</span>, <span class="hljs-string">"facebook/mask2former-swin-large-cityscapes-instance"</span>)
	results = instance_segmentation(image)
	results<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-iszmk4">As you can see below, there are multiple cars classified, and there’s no classification for pixels other than pixels that belong to car and person instances.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'score'</span>: 0.999944,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'car'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.999945,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'car'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.999652,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'car'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.903529,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'person'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-76cxem">Checking out one of the car masks below.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->results[<span class="hljs-number">2</span>][<span class="hljs-string">"mask"</span>]<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-9fwyws"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/instance_segmentation_output.png" alt="Semantic Segmentation Output"></div> <p data-svelte-h="svelte-1oc30s7">Panoptic segmentation combines semantic segmentation and instance segmentation, where every pixel is classified into a class and an instance of that class, and there are multiple masks for each instance of a class. We can use <a href="https://huggingface.co/facebook/mask2former-swin-large-cityscapes-panoptic" rel="nofollow">facebook/mask2former-swin-large-cityscapes-panoptic</a> for this.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->panoptic_segmentation = pipeline(<span class="hljs-string">"image-segmentation"</span>, <span class="hljs-string">"facebook/mask2former-swin-large-cityscapes-panoptic"</span>)
	results = panoptic_segmentation(image)
	results<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13r9j62">As you can see below, we have more classes. We will later illustrate to see that every pixel is classified into one of the classes.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->[{<span class="hljs-string">'score'</span>: 0.999981,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'car'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.999958,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'car'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.99997,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'vegetation'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.999575,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'pole'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.999958,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'building'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.999634,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'road'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.996092,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'sidewalk'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.999221,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'car'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>},
	{<span class="hljs-string">'score'</span>: 0.99987,
	<span class="hljs-string">'label'</span>: <span class="hljs-string">'sky'</span>,
	<span class="hljs-string">'mask'</span>: <PIL.Image.Image image mode=L size=612x415>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-knvn6c">Let’s have a side by side comparison for all types of segmentation.</p> <div class="flex justify-center" data-svelte-h="svelte-6p9cmp"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/segmentation-comparison.png" alt="Segmentation Maps Compared"></div> <p data-svelte-h="svelte-1j96k2e">Seeing all types of segmentation, let’s have a deep dive on fine-tuning a model for semantic segmentation.</p> <p data-svelte-h="svelte-1bguxrb">Common real-world applications of semantic segmentation include training self-driving cars to identify pedestrians and important traffic information, identifying cells and abnormalities in medical imagery, and monitoring environmental changes from satellite imagery.</p> <h2 class="relative group"><a id="fine-tuning-a-model-for-segmentation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fine-tuning-a-model-for-segmentation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Fine-tuning a Model for Segmentation</span></h2> <p data-svelte-h="svelte-7mxw3i">We will now:</p> <ol data-svelte-h="svelte-ofzxxl"><li>Finetune <a href="https://huggingface.co/docs/transformers/main/en/model_doc/segformer#segformer" rel="nofollow">SegFormer</a> on the <a href="https://huggingface.co/datasets/scene_parse_150" rel="nofollow">SceneParse150</a> dataset.</li> <li>Use your fine-tuned model for inference.</li></ol> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-mb1wqs">To see all architectures and checkpoints compatible with this task, we recommend checking the <a href="https://huggingface.co/tasks/image-segmentation" rel="nofollow">task-page</a></p></div> <h3 class="relative group"><a id="load-sceneparse150-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#load-sceneparse150-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Load SceneParse150 dataset</span></h3> <p data-svelte-h="svelte-ldhwp2">Start by loading a smaller subset of the SceneParse150 dataset from the 🤗 Datasets library. This’ll give you a chance to experiment and make sure everything works before spending more time training on the full dataset.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

	<span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">"scene_parse_150"</span>, split=<span class="hljs-string">"train[:50]"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-rugbz4">Split the dataset’s <code>train</code> split into a train and test set with the <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.train_test_split" rel="nofollow">train_test_split</a> method:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>ds = ds.train_test_split(test_size=<span class="hljs-number">0.2</span>)
	<span class="hljs-meta">>>> </span>train_ds = ds[<span class="hljs-string">"train"</span>]
	<span class="hljs-meta">>>> </span>test_ds = ds[<span class="hljs-string">"test"</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1m91ua0">Then take a look at an example:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>train_ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'image'</span>: <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=512x683 at <span class="hljs-number">0x7F9B0C201F90</span>>,
	<span class="hljs-string">'annotation'</span>: <PIL.PngImagePlugin.PngImageFile image mode=L size=512x683 at <span class="hljs-number">0x7F9B0C201DD0</span>>,
	<span class="hljs-string">'scene_category'</span>: <span class="hljs-number">368</span>}

	<span class="hljs-comment"># view the image</span>
	<span class="hljs-meta">>>> </span>train_ds[<span class="hljs-number">0</span>][<span class="hljs-string">"image"</span>]<!-- HTML_TAG_END --></pre></div> <ul data-svelte-h="svelte-1gb3b0f"><li><code>image</code>: a PIL image of the scene.</li> <li><code>annotation</code>: a PIL image of the segmentation map, which is also the model’s target.</li> <li><code>scene_category</code>: a category id that describes the image scene like “kitchen” or “office”. In this guide, you’ll only need <code>image</code> and <code>annotation</code>, both of which are PIL images.</li></ul> <p data-svelte-h="svelte-j46pio">You’ll also want to create a dictionary that maps a label id to a label class which will be useful when you set up the model later. Download the mappings from the Hub and create the <code>id2label</code> and <code>label2id</code> dictionaries:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> json
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hf_hub_download

	<span class="hljs-meta">>>> </span>repo_id = <span class="hljs-string">"huggingface/label-files"</span>
	<span class="hljs-meta">>>> </span>filename = <span class="hljs-string">"ade20k-id2label.json"</span>
	<span class="hljs-meta">>>> </span>id2label = json.loads(Path(hf_hub_download(repo_id, filename, repo_type=<span class="hljs-string">"dataset"</span>)).read_text())
	<span class="hljs-meta">>>> </span>id2label = {<span class="hljs-built_in">int</span>(k): v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> id2label.items()}
	<span class="hljs-meta">>>> </span>label2id = {v: k <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> id2label.items()}
	<span class="hljs-meta">>>> </span>num_labels = <span class="hljs-built_in">len</span>(id2label)<!-- HTML_TAG_END --></pre></div> <h4 class="relative group"><a id="custom-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#custom-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Custom dataset</span></h4> <p data-svelte-h="svelte-1j7tkla">You could also create and use your own dataset if you prefer to train with the <a href="https://github.com/huggingface/transformers/blob/main/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py" rel="nofollow">run_semantic_segmentation.py</a> script instead of a notebook instance. The script requires:</p> <ol><li><p data-svelte-h="svelte-137f4ce">a <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.DatasetDict" rel="nofollow">DatasetDict</a> with two <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Image" rel="nofollow">Image</a> columns, “image” and “label”</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, DatasetDict, Image

	image_paths_train = [<span class="hljs-string">"path/to/image_1.jpg/jpg"</span>, <span class="hljs-string">"path/to/image_2.jpg/jpg"</span>, ..., <span class="hljs-string">"path/to/image_n.jpg/jpg"</span>]
	label_paths_train = [<span class="hljs-string">"path/to/annotation_1.png"</span>, <span class="hljs-string">"path/to/annotation_2.png"</span>, ..., <span class="hljs-string">"path/to/annotation_n.png"</span>]

	image_paths_validation = [...]
	label_paths_validation = [...]

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">create_dataset</span>(<span class="hljs-params">image_paths, label_paths</span>):
	dataset = Dataset.from_dict({<span class="hljs-string">"image"</span>: <span class="hljs-built_in">sorted</span>(image_paths),
	<span class="hljs-string">"label"</span>: <span class="hljs-built_in">sorted</span>(label_paths)})
	dataset = dataset.cast_column(<span class="hljs-string">"image"</span>, Image())
	dataset = dataset.cast_column(<span class="hljs-string">"label"</span>, Image())
	<span class="hljs-keyword">return</span> dataset

	<span class="hljs-comment"># step 1: create Dataset objects</span>
	train_dataset = create_dataset(image_paths_train, label_paths_train)
	validation_dataset = create_dataset(image_paths_validation, label_paths_validation)

	<span class="hljs-comment"># step 2: create DatasetDict</span>
	dataset = DatasetDict({
	<span class="hljs-string">"train"</span>: train_dataset,
	<span class="hljs-string">"validation"</span>: validation_dataset,
	}
	)

	<span class="hljs-comment"># step 3: push to Hub (assumes you have ran the huggingface-cli login command in a terminal/notebook)</span>
	dataset.push_to_hub(<span class="hljs-string">"your-name/dataset-repo"</span>)

	<span class="hljs-comment"># optionally, you can push to a private repo on the Hub</span>
	<span class="hljs-comment"># dataset.push_to_hub("name of repo on the hub", private=True)</span><!-- HTML_TAG_END --></pre></div></li> <li><p data-svelte-h="svelte-1je4dsn">an id2label dictionary mapping the class integers to their class names</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> json
	<span class="hljs-comment"># simple example</span>
	id2label = {<span class="hljs-number">0</span>: <span class="hljs-string">'cat'</span>, <span class="hljs-number">1</span>: <span class="hljs-string">'dog'</span>}
	<span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">'id2label.json'</span>, <span class="hljs-string">'w'</span>) <span class="hljs-keyword">as</span> fp:
	json.dump(id2label, fp)<!-- HTML_TAG_END --></pre></div></li></ol> <p data-svelte-h="svelte-1k7ggj5">As an example, take a look at this <a href="https://huggingface.co/datasets/nielsr/ade20k-demo" rel="nofollow">example dataset</a> which was created with the steps shown above.</p> <h3 class="relative group"><a id="preprocess" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#preprocess"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Preprocess</span></h3> <p data-svelte-h="svelte-1wntzlt">The next step is to load a SegFormer image processor to prepare the images and annotations for the model. Some datasets, like this one, use the zero-index as the background class. However, the background class isn’t actually included in the 150 classes, so you’ll need to set <code>do_reduce_labels=True</code> to subtract one from all the labels. The zero-index is replaced by <code>255</code> so it’s ignored by SegFormer’s loss function:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoImageProcessor

	<span class="hljs-meta">>>> </span>checkpoint = <span class="hljs-string">"nvidia/mit-b0"</span>
	<span class="hljs-meta">>>> </span>image_processor = AutoImageProcessor.from_pretrained(checkpoint, do_reduce_labels=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg> <span>Pytorch</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide Pytorch content</span></div></div> <div class="framework-content"> <p data-svelte-h="svelte-43dxhy">It is common to apply some data augmentations to an image dataset to make a model more robust against overfitting. In this guide, you’ll use the <a href="https://pytorch.org/vision/stable/generated/torchvision.transforms.ColorJitter.html" rel="nofollow"><code>ColorJitter</code></a> function from <a href="https://pytorch.org/vision/stable/index.html" rel="nofollow">torchvision</a> to randomly change the color properties of an image, but you can also use any image library you like.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> ColorJitter

	<span class="hljs-meta">>>> </span>jitter = ColorJitter(brightness=<span class="hljs-number">0.25</span>, contrast=<span class="hljs-number">0.25</span>, saturation=<span class="hljs-number">0.25</span>, hue=<span class="hljs-number">0.1</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-qerdt3">Now create two preprocessing functions to prepare the images and annotations for the model. These functions convert the images into <code>pixel_values</code> and annotations to <code>labels</code>. For the training set, <code>jitter</code> is applied before providing the images to the image processor. For the test set, the image processor crops and normalizes the <code>images</code>, and only crops the <code>labels</code> because no data augmentation is applied during testing.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">train_transforms</span>(<span class="hljs-params">example_batch</span>):
	<span class="hljs-meta">... </span> images = [jitter(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">"image"</span>]]
	<span class="hljs-meta">... </span> labels = [x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">"annotation"</span>]]
	<span class="hljs-meta">... </span> inputs = image_processor(images, labels)
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> inputs


	<span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">val_transforms</span>(<span class="hljs-params">example_batch</span>):
	<span class="hljs-meta">... </span> images = [x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">"image"</span>]]
	<span class="hljs-meta">... </span> labels = [x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">"annotation"</span>]]
	<span class="hljs-meta">... </span> inputs = image_processor(images, labels)
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> inputs<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-tijv00">To apply the <code>jitter</code> over the entire dataset, use the 🤗 Datasets <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.set_transform" rel="nofollow">set_transform</a> function. The transform is applied on the fly which is faster and consumes less disk space:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>train_ds.set_transform(train_transforms)
	<span class="hljs-meta">>>> </span>test_ds.set_transform(val_transforms)<!-- HTML_TAG_END --></pre></div></div></div> </div> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="0.94em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 274"><path d="M145.726 42.065v42.07l72.861 42.07v-42.07l-72.86-42.07zM0 84.135v42.07l36.43 21.03V105.17L0 84.135zm109.291 21.035l-36.43 21.034v126.2l36.43 21.035v-84.135l36.435 21.035v-42.07l-36.435-21.034V105.17z" fill="#E55B2D"></path><path d="M145.726 42.065L36.43 105.17v42.065l72.861-42.065v42.065l36.435-21.03v-84.14zM255.022 63.1l-36.435 21.035v42.07l36.435-21.035V63.1zm-72.865 84.135l-36.43 21.035v42.07l36.43-21.036v-42.07zm-36.43 63.104l-36.436-21.035v84.135l36.435-21.035V210.34z" fill="#ED8E24"></path><path d="M145.726 0L0 84.135l36.43 21.035l109.296-63.105l72.861 42.07L255.022 63.1L145.726 0zm0 126.204l-36.435 21.03l36.435 21.036l36.43-21.035l-36.43-21.03z" fill="#F8BF3C"></path></svg> <span>TensorFlow</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide TensorFlow content</span></div></div> <div class="framework-content"> <p data-svelte-h="svelte-1rj6vla">It is common to apply some data augmentations to an image dataset to make a model more robust against overfitting.
	In this guide, you’ll use <a href="https://www.tensorflow.org/api_docs/python/tf/image" rel="nofollow"><code>tf.image</code></a> to randomly change the color properties of an image, but you can also use any image
	library you like.
	Define two separate transformation functions:</p> <ul data-svelte-h="svelte-lmmfid"><li>training data transformations that include image augmentation</li> <li>validation data transformations that only transpose the images, since computer vision models in 🤗 Transformers expect channels-first layout</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf


	<span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">aug_transforms</span>(<span class="hljs-params">image</span>):
	<span class="hljs-meta">... </span> image = tf.keras.utils.img_to_array(image)
	<span class="hljs-meta">... </span> image = tf.image.random_brightness(image, <span class="hljs-number">0.25</span>)
	<span class="hljs-meta">... </span> image = tf.image.random_contrast(image, <span class="hljs-number">0.5</span>, <span class="hljs-number">2.0</span>)
	<span class="hljs-meta">... </span> image = tf.image.random_saturation(image, <span class="hljs-number">0.75</span>, <span class="hljs-number">1.25</span>)
	<span class="hljs-meta">... </span> image = tf.image.random_hue(image, <span class="hljs-number">0.1</span>)
	<span class="hljs-meta">... </span> image = tf.transpose(image, (<span class="hljs-number">2</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>))
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> image


	<span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">image</span>):
	<span class="hljs-meta">... </span> image = tf.keras.utils.img_to_array(image)
	<span class="hljs-meta">... </span> image = tf.transpose(image, (<span class="hljs-number">2</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>))
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> image<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-hpi41l">Next, create two preprocessing functions to prepare batches of images and annotations for the model. These functions apply
	the image transformations and use the earlier loaded <code>image_processor</code> to convert the images into <code>pixel_values</code> and
	annotations to <code>labels</code>. <code>ImageProcessor</code> also takes care of resizing and normalizing the images.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">train_transforms</span>(<span class="hljs-params">example_batch</span>):
	<span class="hljs-meta">... </span> images = [aug_transforms(x.convert(<span class="hljs-string">"RGB"</span>)) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">"image"</span>]]
	<span class="hljs-meta">... </span> labels = [x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">"annotation"</span>]]
	<span class="hljs-meta">... </span> inputs = image_processor(images, labels)
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> inputs


	<span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">val_transforms</span>(<span class="hljs-params">example_batch</span>):
	<span class="hljs-meta">... </span> images = [transforms(x.convert(<span class="hljs-string">"RGB"</span>)) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">"image"</span>]]
	<span class="hljs-meta">... </span> labels = [x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> example_batch[<span class="hljs-string">"annotation"</span>]]
	<span class="hljs-meta">... </span> inputs = image_processor(images, labels)
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> inputs<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-19lox7v">To apply the preprocessing transformations over the entire dataset, use the 🤗 Datasets <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.set_transform" rel="nofollow">set_transform</a> function.
	The transform is applied on the fly which is faster and consumes less disk space:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>train_ds.set_transform(train_transforms)
	<span class="hljs-meta">>>> </span>test_ds.set_transform(val_transforms)<!-- HTML_TAG_END --></pre></div></div></div> </div> <h3 class="relative group"><a id="evaluate" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#evaluate"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Evaluate</span></h3> <p data-svelte-h="svelte-vg2xml">Including a metric during training is often helpful for evaluating your model’s performance. You can quickly load an evaluation method with the 🤗 <a href="https://huggingface.co/docs/evaluate/index" rel="nofollow">Evaluate</a> library. For this task, load the <a href="https://huggingface.co/spaces/evaluate-metric/accuracy" rel="nofollow">mean Intersection over Union</a> (IoU) metric (see the 🤗 Evaluate <a href="https://huggingface.co/docs/evaluate/a_quick_tour" rel="nofollow">quick tour</a> to learn more about how to load and compute a metric):</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> evaluate

	<span class="hljs-meta">>>> </span>metric = evaluate.load(<span class="hljs-string">"mean_iou"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-aq72n8">Then create a function to <a href="https://huggingface.co/docs/evaluate/main/en/package_reference/main_classes#evaluate.EvaluationModule.compute" rel="nofollow">compute</a> the metrics. Your predictions need to be converted to
	logits first, and then reshaped to match the size of the labels before you can call <a href="https://huggingface.co/docs/evaluate/main/en/package_reference/main_classes#evaluate.EvaluationModule.compute" rel="nofollow">compute</a>:</p> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg> <span>Pytorch</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide Pytorch content</span></div></div> <div class="framework-content"> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn

	<span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_pred</span>):
	<span class="hljs-meta">... </span> <span class="hljs-keyword">with</span> torch.no_grad():
	<span class="hljs-meta">... </span> logits, labels = eval_pred
	<span class="hljs-meta">... </span> logits_tensor = torch.from_numpy(logits)
	<span class="hljs-meta">... </span> logits_tensor = nn.functional.interpolate(
	<span class="hljs-meta">... </span> logits_tensor,
	<span class="hljs-meta">... </span> size=labels.shape[-<span class="hljs-number">2</span>:],
	<span class="hljs-meta">... </span> mode=<span class="hljs-string">"bilinear"</span>,
	<span class="hljs-meta">... </span> align_corners=<span class="hljs-literal">False</span>,
	<span class="hljs-meta">... </span> ).argmax(dim=<span class="hljs-number">1</span>)

	<span class="hljs-meta">... </span> pred_labels = logits_tensor.detach().cpu().numpy()
	<span class="hljs-meta">... </span> metrics = metric.compute(
	<span class="hljs-meta">... </span> predictions=pred_labels,
	<span class="hljs-meta">... </span> references=labels,
	<span class="hljs-meta">... </span> num_labels=num_labels,
	<span class="hljs-meta">... </span> ignore_index=<span class="hljs-number">255</span>,
	<span class="hljs-meta">... </span> reduce_labels=<span class="hljs-literal">False</span>,
	<span class="hljs-meta">... </span> )
	<span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> key, value <span class="hljs-keyword">in</span> metrics.items():
	<span class="hljs-meta">... </span> <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(value, np.ndarray):
	<span class="hljs-meta">... </span> metrics[key] = value.tolist()
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> metrics<!-- HTML_TAG_END --></pre></div></div></div> </div> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="0.94em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 274"><path d="M145.726 42.065v42.07l72.861 42.07v-42.07l-72.86-42.07zM0 84.135v42.07l36.43 21.03V105.17L0 84.135zm109.291 21.035l-36.43 21.034v126.2l36.43 21.035v-84.135l36.435 21.035v-42.07l-36.435-21.034V105.17z" fill="#E55B2D"></path><path d="M145.726 42.065L36.43 105.17v42.065l72.861-42.065v42.065l36.435-21.03v-84.14zM255.022 63.1l-36.435 21.035v42.07l36.435-21.035V63.1zm-72.865 84.135l-36.43 21.035v42.07l36.43-21.036v-42.07zm-36.43 63.104l-36.436-21.035v84.135l36.435-21.035V210.34z" fill="#ED8E24"></path><path d="M145.726 0L0 84.135l36.43 21.035l109.296-63.105l72.861 42.07L255.022 63.1L145.726 0zm0 126.204l-36.435 21.03l36.435 21.036l36.43-21.035l-36.43-21.03z" fill="#F8BF3C"></path></svg> <span>TensorFlow</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide TensorFlow content</span></div></div> <div class="framework-content"> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_pred</span>):
	<span class="hljs-meta">... </span> logits, labels = eval_pred
	<span class="hljs-meta">... </span> logits = tf.transpose(logits, perm=[<span class="hljs-number">0</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">1</span>])
	<span class="hljs-meta">... </span> logits_resized = tf.image.resize(
	<span class="hljs-meta">... </span> logits,
	<span class="hljs-meta">... </span> size=tf.shape(labels)[<span class="hljs-number">1</span>:],
	<span class="hljs-meta">... </span> method=<span class="hljs-string">"bilinear"</span>,
	<span class="hljs-meta">... </span> )

	<span class="hljs-meta">... </span> pred_labels = tf.argmax(logits_resized, axis=-<span class="hljs-number">1</span>)
	<span class="hljs-meta">... </span> metrics = metric.compute(
	<span class="hljs-meta">... </span> predictions=pred_labels,
	<span class="hljs-meta">... </span> references=labels,
	<span class="hljs-meta">... </span> num_labels=num_labels,
	<span class="hljs-meta">... </span> ignore_index=-<span class="hljs-number">1</span>,
	<span class="hljs-meta">... </span> reduce_labels=image_processor.do_reduce_labels,
	<span class="hljs-meta">... </span> )

	<span class="hljs-meta">... </span> per_category_accuracy = metrics.pop(<span class="hljs-string">"per_category_accuracy"</span>).tolist()
	<span class="hljs-meta">... </span> per_category_iou = metrics.pop(<span class="hljs-string">"per_category_iou"</span>).tolist()

	<span class="hljs-meta">... </span> metrics.update({<span class="hljs-string">f"accuracy_<span class="hljs-subst">{id2label[i]}</span>"</span>: v <span class="hljs-keyword">for</span> i, v <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(per_category_accuracy)})
	<span class="hljs-meta">... </span> metrics.update({<span class="hljs-string">f"iou_<span class="hljs-subst">{id2label[i]}</span>"</span>: v <span class="hljs-keyword">for</span> i, v <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(per_category_iou)})
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> {<span class="hljs-string">"val_"</span> + k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> metrics.items()}<!-- HTML_TAG_END --></pre></div></div></div> </div> <p data-svelte-h="svelte-183aynn">Your <code>compute_metrics</code> function is ready to go now, and you’ll return to it when you setup your training.</p> <h3 class="relative group"><a id="train" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#train"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Train</span></h3> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg> <span>Pytorch</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide Pytorch content</span></div></div> <div class="framework-content"> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-11yuerp">If you aren’t familiar with finetuning a model with the <a href="/docs/transformers/pr_33913/en/main_classes/trainer#transformers.Trainer">Trainer</a>, take a look at the basic tutorial <a href="../training#finetune-with-trainer">here</a>!</p></div> <p data-svelte-h="svelte-1icxqjx">You’re ready to start training your model now! Load SegFormer with <a href="/docs/transformers/pr_33913/en/model_doc/auto#transformers.AutoModelForSemanticSegmentation">AutoModelForSemanticSegmentation</a>, and pass the model the mapping between label ids and label classes:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSemanticSegmentation, TrainingArguments, Trainer

	<span class="hljs-meta">>>> </span>model = AutoModelForSemanticSegmentation.from_pretrained(checkpoint, id2label=id2label, label2id=label2id)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-l42k0i">At this point, only three steps remain:</p> <ol data-svelte-h="svelte-1lsf9tg"><li>Define your training hyperparameters in <a href="/docs/transformers/pr_33913/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>. It is important you don’t remove unused columns because this’ll drop the <code>image</code> column. Without the <code>image</code> column, you can’t create <code>pixel_values</code>. Set <code>remove_unused_columns=False</code> to prevent this behavior! The only other required parameter is <code>output_dir</code> which specifies where to save your model. You’ll push this model to the Hub by setting <code>push_to_hub=True</code> (you need to be signed in to Hugging Face to upload your model). At the end of each epoch, the <a href="/docs/transformers/pr_33913/en/main_classes/trainer#transformers.Trainer">Trainer</a> will evaluate the IoU metric and save the training checkpoint.</li> <li>Pass the training arguments to <a href="/docs/transformers/pr_33913/en/main_classes/trainer#transformers.Trainer">Trainer</a> along with the model, dataset, tokenizer, data collator, and <code>compute_metrics</code> function.</li> <li>Call <a href="/docs/transformers/pr_33913/en/main_classes/trainer#transformers.Trainer.train">train()</a> to finetune your model.</li></ol> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>training_args = TrainingArguments(
	<span class="hljs-meta">... </span> output_dir=<span class="hljs-string">"segformer-b0-scene-parse-150"</span>,
	<span class="hljs-meta">... </span> learning_rate=<span class="hljs-number">6e-5</span>,
	<span class="hljs-meta">... </span> num_train_epochs=<span class="hljs-number">50</span>,
	<span class="hljs-meta">... </span> per_device_train_batch_size=<span class="hljs-number">2</span>,
	<span class="hljs-meta">... </span> per_device_eval_batch_size=<span class="hljs-number">2</span>,
	<span class="hljs-meta">... </span> save_total_limit=<span class="hljs-number">3</span>,
	<span class="hljs-meta">... </span> eval_strategy=<span class="hljs-string">"steps"</span>,
	<span class="hljs-meta">... </span> save_strategy=<span class="hljs-string">"steps"</span>,
	<span class="hljs-meta">... </span> save_steps=<span class="hljs-number">20</span>,
	<span class="hljs-meta">... </span> eval_steps=<span class="hljs-number">20</span>,
	<span class="hljs-meta">... </span> logging_steps=<span class="hljs-number">1</span>,
	<span class="hljs-meta">... </span> eval_accumulation_steps=<span class="hljs-number">5</span>,
	<span class="hljs-meta">... </span> remove_unused_columns=<span class="hljs-literal">False</span>,
	<span class="hljs-meta">... </span> push_to_hub=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>trainer = Trainer(
	<span class="hljs-meta">... </span> model=model,
	<span class="hljs-meta">... </span> args=training_args,
	<span class="hljs-meta">... </span> train_dataset=train_ds,
	<span class="hljs-meta">... </span> eval_dataset=test_ds,
	<span class="hljs-meta">... </span> compute_metrics=compute_metrics,
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>trainer.train()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-t7oi65">Once training is completed, share your model to the Hub with the <a href="/docs/transformers/pr_33913/en/main_classes/trainer#transformers.Trainer.push_to_hub">push_to_hub()</a> method so everyone can use your model:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>trainer.push_to_hub()<!-- HTML_TAG_END --></pre></div></div></div> </div> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="0.94em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 274"><path d="M145.726 42.065v42.07l72.861 42.07v-42.07l-72.86-42.07zM0 84.135v42.07l36.43 21.03V105.17L0 84.135zm109.291 21.035l-36.43 21.034v126.2l36.43 21.035v-84.135l36.435 21.035v-42.07l-36.435-21.034V105.17z" fill="#E55B2D"></path><path d="M145.726 42.065L36.43 105.17v42.065l72.861-42.065v42.065l36.435-21.03v-84.14zM255.022 63.1l-36.435 21.035v42.07l36.435-21.035V63.1zm-72.865 84.135l-36.43 21.035v42.07l36.43-21.036v-42.07zm-36.43 63.104l-36.436-21.035v84.135l36.435-21.035V210.34z" fill="#ED8E24"></path><path d="M145.726 0L0 84.135l36.43 21.035l109.296-63.105l72.861 42.07L255.022 63.1L145.726 0zm0 126.204l-36.435 21.03l36.435 21.036l36.43-21.035l-36.43-21.03z" fill="#F8BF3C"></path></svg> <span>TensorFlow</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide TensorFlow content</span></div></div> <div class="framework-content"> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1egt5s9">If you are unfamiliar with fine-tuning a model with Keras, check out the <a href="./training#train-a-tensorflow-model-with-keras">basic tutorial</a> first!</p></div> <p data-svelte-h="svelte-s07fxj">To fine-tune a model in TensorFlow, follow these steps:</p> <ol data-svelte-h="svelte-1e3w1hr"><li>Define the training hyperparameters, and set up an optimizer and a learning rate schedule.</li> <li>Instantiate a pretrained model.</li> <li>Convert a 🤗 Dataset to a <code>tf.data.Dataset</code>.</li> <li>Compile your model.</li> <li>Add callbacks to calculate metrics and upload your model to 🤗 Hub</li> <li>Use the <code>fit()</code> method to run the training.</li></ol> <p data-svelte-h="svelte-ccl3wn">Start by defining the hyperparameters, optimizer and learning rate schedule:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

	<span class="hljs-meta">>>> </span>batch_size = <span class="hljs-number">2</span>
	<span class="hljs-meta">>>> </span>num_epochs = <span class="hljs-number">50</span>
	<span class="hljs-meta">>>> </span>num_train_steps = <span class="hljs-built_in">len</span>(train_ds) * num_epochs
	<span class="hljs-meta">>>> </span>learning_rate = <span class="hljs-number">6e-5</span>
	<span class="hljs-meta">>>> </span>weight_decay_rate = <span class="hljs-number">0.01</span>

	<span class="hljs-meta">>>> </span>optimizer, lr_schedule = create_optimizer(
	<span class="hljs-meta">... </span> init_lr=learning_rate,
	<span class="hljs-meta">... </span> num_train_steps=num_train_steps,
	<span class="hljs-meta">... </span> weight_decay_rate=weight_decay_rate,
	<span class="hljs-meta">... </span> num_warmup_steps=<span class="hljs-number">0</span>,
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1tg22db">Then, load SegFormer with <a href="/docs/transformers/pr_33913/en/model_doc/auto#transformers.TFAutoModelForSemanticSegmentation">TFAutoModelForSemanticSegmentation</a> along with the label mappings, and compile it with the
	optimizer. Note that Transformers models all have a default task-relevant loss function, so you don’t need to specify one unless you want to:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSemanticSegmentation

	<span class="hljs-meta">>>> </span>model = TFAutoModelForSemanticSegmentation.from_pretrained(
	<span class="hljs-meta">... </span> checkpoint,
	<span class="hljs-meta">... </span> id2label=id2label,
	<span class="hljs-meta">... </span> label2id=label2id,
	<span class="hljs-meta">... </span>)
	<span class="hljs-meta">>>> </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer) <span class="hljs-comment"># No loss argument!</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1wjpyah">Convert your datasets to the <code>tf.data.Dataset</code> format using the <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset" rel="nofollow">to_tf_dataset</a> and the <a href="/docs/transformers/pr_33913/en/main_classes/data_collator#transformers.DefaultDataCollator">DefaultDataCollator</a>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

	<span class="hljs-meta">>>> </span>data_collator = DefaultDataCollator(return_tensors=<span class="hljs-string">"tf"</span>)

	<span class="hljs-meta">>>> </span>tf_train_dataset = train_ds.to_tf_dataset(
	<span class="hljs-meta">... </span> columns=[<span class="hljs-string">"pixel_values"</span>, <span class="hljs-string">"label"</span>],
	<span class="hljs-meta">... </span> shuffle=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> batch_size=batch_size,
	<span class="hljs-meta">... </span> collate_fn=data_collator,
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>tf_eval_dataset = test_ds.to_tf_dataset(
	<span class="hljs-meta">... </span> columns=[<span class="hljs-string">"pixel_values"</span>, <span class="hljs-string">"label"</span>],
	<span class="hljs-meta">... </span> shuffle=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> batch_size=batch_size,
	<span class="hljs-meta">... </span> collate_fn=data_collator,
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1bhxpnh">To compute the accuracy from the predictions and push your model to the 🤗 Hub, use <a href="../main_classes/keras_callbacks">Keras callbacks</a>.
	Pass your <code>compute_metrics</code> function to <a href="/docs/transformers/pr_33913/en/main_classes/keras_callbacks#transformers.KerasMetricCallback">KerasMetricCallback</a>,
	and use the <a href="/docs/transformers/pr_33913/en/main_classes/keras_callbacks#transformers.PushToHubCallback">PushToHubCallback</a> to upload the model:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> KerasMetricCallback, PushToHubCallback

	<span class="hljs-meta">>>> </span>metric_callback = KerasMetricCallback(
	<span class="hljs-meta">... </span> metric_fn=compute_metrics, eval_dataset=tf_eval_dataset, batch_size=batch_size, label_cols=[<span class="hljs-string">"labels"</span>]
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>push_to_hub_callback = PushToHubCallback(output_dir=<span class="hljs-string">"scene_segmentation"</span>, tokenizer=image_processor)

	<span class="hljs-meta">>>> </span>callbacks = [metric_callback, push_to_hub_callback]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1occr1z">Finally, you are ready to train your model! Call <code>fit()</code> with your training and validation datasets, the number of epochs,
	and your callbacks to fine-tune the model:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>model.fit(
	<span class="hljs-meta">... </span> tf_train_dataset,
	<span class="hljs-meta">... </span> validation_data=tf_eval_dataset,
	<span class="hljs-meta">... </span> callbacks=callbacks,
	<span class="hljs-meta">... </span> epochs=num_epochs,
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1r99pbn">Congratulations! You have fine-tuned your model and shared it on the 🤗 Hub. You can now use it for inference!</p></div></div> </div> <h3 class="relative group"><a id="inference" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inference"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inference</span></h3> <p data-svelte-h="svelte-633ppb">Great, now that you’ve finetuned a model, you can use it for inference!</p> <p data-svelte-h="svelte-13x04z1">Reload the dataset and load an image for inference.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

	<span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">"scene_parse_150"</span>, split=<span class="hljs-string">"train[:50]"</span>)
	<span class="hljs-meta">>>> </span>ds = ds.train_test_split(test_size=<span class="hljs-number">0.2</span>)
	<span class="hljs-meta">>>> </span>test_ds = ds[<span class="hljs-string">"test"</span>]
	<span class="hljs-meta">>>> </span>image = ds[<span class="hljs-string">"test"</span>][<span class="hljs-number">0</span>][<span class="hljs-string">"image"</span>]
	<span class="hljs-meta">>>> </span>image<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-11jfm1f"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/semantic-seg-image.png" alt="Image of bedroom"></div> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg> <span>Pytorch</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide Pytorch content</span></div></div> <div class="framework-content"> <p data-svelte-h="svelte-appsul">We will now see how to infer without a pipeline. Process the image with an image processor and place the <code>pixel_values</code> on a GPU:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend
	<span class="hljs-comment"># automatically detects the underlying device type (CUDA, CPU, XPU, MPS, etc.)</span>
	<span class="hljs-meta">>>> </span>device, _, _ = get_backend()
	<span class="hljs-meta">>>> </span>encoding = image_processor(image, return_tensors=<span class="hljs-string">"pt"</span>)
	<span class="hljs-meta">>>> </span>pixel_values = encoding.pixel_values.to(device)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-oyplyw">Pass your input to the model and return the <code>logits</code>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>outputs = model(pixel_values=pixel_values)
	<span class="hljs-meta">>>> </span>logits = outputs.logits.cpu()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-tk6q3q">Next, rescale the logits to the original image size:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>upsampled_logits = nn.functional.interpolate(
	<span class="hljs-meta">... </span> logits,
	<span class="hljs-meta">... </span> size=image.size[::-<span class="hljs-number">1</span>],
	<span class="hljs-meta">... </span> mode=<span class="hljs-string">"bilinear"</span>,
	<span class="hljs-meta">... </span> align_corners=<span class="hljs-literal">False</span>,
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>pred_seg = upsampled_logits.argmax(dim=<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div></div></div> </div> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="0.94em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 274"><path d="M145.726 42.065v42.07l72.861 42.07v-42.07l-72.86-42.07zM0 84.135v42.07l36.43 21.03V105.17L0 84.135zm109.291 21.035l-36.43 21.034v126.2l36.43 21.035v-84.135l36.435 21.035v-42.07l-36.435-21.034V105.17z" fill="#E55B2D"></path><path d="M145.726 42.065L36.43 105.17v42.065l72.861-42.065v42.065l36.435-21.03v-84.14zM255.022 63.1l-36.435 21.035v42.07l36.435-21.035V63.1zm-72.865 84.135l-36.43 21.035v42.07l36.43-21.036v-42.07zm-36.43 63.104l-36.436-21.035v84.135l36.435-21.035V210.34z" fill="#ED8E24"></path><path d="M145.726 0L0 84.135l36.43 21.035l109.296-63.105l72.861 42.07L255.022 63.1L145.726 0zm0 126.204l-36.435 21.03l36.435 21.036l36.43-21.035l-36.43-21.03z" fill="#F8BF3C"></path></svg> <span>TensorFlow</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide TensorFlow content</span></div></div> <div class="framework-content"> <p data-svelte-h="svelte-1r2yss0">Load an image processor to preprocess the image and return the input as TensorFlow tensors:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoImageProcessor

	<span class="hljs-meta">>>> </span>image_processor = AutoImageProcessor.from_pretrained(<span class="hljs-string">"MariaK/scene_segmentation"</span>)
	<span class="hljs-meta">>>> </span>inputs = image_processor(image, return_tensors=<span class="hljs-string">"tf"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-oyplyw">Pass your input to the model and return the <code>logits</code>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSemanticSegmentation

	<span class="hljs-meta">>>> </span>model = TFAutoModelForSemanticSegmentation.from_pretrained(<span class="hljs-string">"MariaK/scene_segmentation"</span>)
	<span class="hljs-meta">>>> </span>logits = model(**inputs).logits<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-enih9r">Next, rescale the logits to the original image size and apply argmax on the class dimension:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>logits = tf.transpose(logits, [<span class="hljs-number">0</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">1</span>])

	<span class="hljs-meta">>>> </span>upsampled_logits = tf.image.resize(
	<span class="hljs-meta">... </span> logits,
	<span class="hljs-meta">... </span> <span class="hljs-comment"># We reverse the shape of `image` because `image.size` returns width and height.</span>
	<span class="hljs-meta">... </span> image.size[::-<span class="hljs-number">1</span>],
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>pred_seg = tf.math.argmax(upsampled_logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div></div></div> </div> <p data-svelte-h="svelte-9v6c94">To visualize the results, load the <a href="https://github.com/tensorflow/models/blob/3f1ca33afe3c1631b733ea7e40c294273b9e406d/research/deeplab/utils/get_dataset_colormap.py#L51" rel="nofollow">dataset color palette</a> as <code>ade_palette()</code> that maps each class to their RGB values.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">ade_palette</span>():
	<span class="hljs-keyword">return</span> np.asarray([
	[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">120</span>, <span class="hljs-number">120</span>, <span class="hljs-number">120</span>],
	[<span class="hljs-number">180</span>, <span class="hljs-number">120</span>, <span class="hljs-number">120</span>],
	[<span class="hljs-number">6</span>, <span class="hljs-number">230</span>, <span class="hljs-number">230</span>],
	[<span class="hljs-number">80</span>, <span class="hljs-number">50</span>, <span class="hljs-number">50</span>],
	[<span class="hljs-number">4</span>, <span class="hljs-number">200</span>, <span class="hljs-number">3</span>],
	[<span class="hljs-number">120</span>, <span class="hljs-number">120</span>, <span class="hljs-number">80</span>],
	[<span class="hljs-number">140</span>, <span class="hljs-number">140</span>, <span class="hljs-number">140</span>],
	[<span class="hljs-number">204</span>, <span class="hljs-number">5</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">230</span>, <span class="hljs-number">230</span>, <span class="hljs-number">230</span>],
	[<span class="hljs-number">4</span>, <span class="hljs-number">250</span>, <span class="hljs-number">7</span>],
	[<span class="hljs-number">224</span>, <span class="hljs-number">5</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">235</span>, <span class="hljs-number">255</span>, <span class="hljs-number">7</span>],
	[<span class="hljs-number">150</span>, <span class="hljs-number">5</span>, <span class="hljs-number">61</span>],
	[<span class="hljs-number">120</span>, <span class="hljs-number">120</span>, <span class="hljs-number">70</span>],
	[<span class="hljs-number">8</span>, <span class="hljs-number">255</span>, <span class="hljs-number">51</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">6</span>, <span class="hljs-number">82</span>],
	[<span class="hljs-number">143</span>, <span class="hljs-number">255</span>, <span class="hljs-number">140</span>],
	[<span class="hljs-number">204</span>, <span class="hljs-number">255</span>, <span class="hljs-number">4</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">51</span>, <span class="hljs-number">7</span>],
	[<span class="hljs-number">204</span>, <span class="hljs-number">70</span>, <span class="hljs-number">3</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">102</span>, <span class="hljs-number">200</span>],
	[<span class="hljs-number">61</span>, <span class="hljs-number">230</span>, <span class="hljs-number">250</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">6</span>, <span class="hljs-number">51</span>],
	[<span class="hljs-number">11</span>, <span class="hljs-number">102</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">7</span>, <span class="hljs-number">71</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">9</span>, <span class="hljs-number">224</span>],
	[<span class="hljs-number">9</span>, <span class="hljs-number">7</span>, <span class="hljs-number">230</span>],
	[<span class="hljs-number">220</span>, <span class="hljs-number">220</span>, <span class="hljs-number">220</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">9</span>, <span class="hljs-number">92</span>],
	[<span class="hljs-number">112</span>, <span class="hljs-number">9</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">8</span>, <span class="hljs-number">255</span>, <span class="hljs-number">214</span>],
	[<span class="hljs-number">7</span>, <span class="hljs-number">255</span>, <span class="hljs-number">224</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">184</span>, <span class="hljs-number">6</span>],
	[<span class="hljs-number">10</span>, <span class="hljs-number">255</span>, <span class="hljs-number">71</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">41</span>, <span class="hljs-number">10</span>],
	[<span class="hljs-number">7</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">224</span>, <span class="hljs-number">255</span>, <span class="hljs-number">8</span>],
	[<span class="hljs-number">102</span>, <span class="hljs-number">8</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">61</span>, <span class="hljs-number">6</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">194</span>, <span class="hljs-number">7</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">122</span>, <span class="hljs-number">8</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">20</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">8</span>, <span class="hljs-number">41</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">5</span>, <span class="hljs-number">153</span>],
	[<span class="hljs-number">6</span>, <span class="hljs-number">51</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">235</span>, <span class="hljs-number">12</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">160</span>, <span class="hljs-number">150</span>, <span class="hljs-number">20</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">163</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">140</span>, <span class="hljs-number">140</span>, <span class="hljs-number">140</span>],
	[<span class="hljs-number">250</span>, <span class="hljs-number">10</span>, <span class="hljs-number">15</span>],
	[<span class="hljs-number">20</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">31</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">31</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">224</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">153</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">71</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">235</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">173</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">31</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">11</span>, <span class="hljs-number">200</span>, <span class="hljs-number">200</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">82</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">245</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">61</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">112</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">133</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">163</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">194</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">143</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">51</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">82</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">41</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">173</span>],
	[<span class="hljs-number">10</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">173</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">153</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">92</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">245</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">102</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">173</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">20</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">184</span>, <span class="hljs-number">184</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">31</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">61</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">71</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">204</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">194</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">82</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">10</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">112</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">51</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">194</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">122</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">163</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">153</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">10</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">112</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">143</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">82</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">163</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">235</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">8</span>, <span class="hljs-number">184</span>, <span class="hljs-number">170</span>],
	[<span class="hljs-number">133</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">92</span>],
	[<span class="hljs-number">184</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">31</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">184</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">214</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">112</span>],
	[<span class="hljs-number">92</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">224</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">112</span>, <span class="hljs-number">224</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">70</span>, <span class="hljs-number">184</span>, <span class="hljs-number">160</span>],
	[<span class="hljs-number">163</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">153</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">71</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">163</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">204</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">143</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">235</span>],
	[<span class="hljs-number">133</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">235</span>],
	[<span class="hljs-number">245</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">0</span>, <span class="hljs-number">122</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">245</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">10</span>, <span class="hljs-number">190</span>, <span class="hljs-number">212</span>],
	[<span class="hljs-number">214</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">204</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">20</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">153</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">41</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">204</span>],
	[<span class="hljs-number">41</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">41</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">173</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">245</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">71</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">122</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">255</span>, <span class="hljs-number">184</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">92</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">184</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">0</span>, <span class="hljs-number">133</span>, <span class="hljs-number">255</span>],
	[<span class="hljs-number">255</span>, <span class="hljs-number">214</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">25</span>, <span class="hljs-number">194</span>, <span class="hljs-number">194</span>],
	[<span class="hljs-number">102</span>, <span class="hljs-number">255</span>, <span class="hljs-number">0</span>],
	[<span class="hljs-number">92</span>, <span class="hljs-number">0</span>, <span class="hljs-number">255</span>],
	])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-93zsmv">Then you can combine and plot your image and the predicted segmentation map:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

	<span class="hljs-meta">>>> </span>color_seg = np.zeros((pred_seg.shape[<span class="hljs-number">0</span>], pred_seg.shape[<span class="hljs-number">1</span>], <span class="hljs-number">3</span>), dtype=np.uint8)
	<span class="hljs-meta">>>> </span>palette = np.array(ade_palette())
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> label, color <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(palette):
	<span class="hljs-meta">... </span> color_seg[pred_seg == label, :] = color
	<span class="hljs-meta">>>> </span>color_seg = color_seg[..., ::-<span class="hljs-number">1</span>] <span class="hljs-comment"># convert to BGR</span>

	<span class="hljs-meta">>>> </span>img = np.array(image) * <span class="hljs-number">0.5</span> + color_seg * <span class="hljs-number">0.5</span> <span class="hljs-comment"># plot the image with the segmentation map</span>
	<span class="hljs-meta">>>> </span>img = img.astype(np.uint8)

	<span class="hljs-meta">>>> </span>plt.figure(figsize=(<span class="hljs-number">15</span>, <span class="hljs-number">10</span>))
	<span class="hljs-meta">>>> </span>plt.imshow(img)
	<span class="hljs-meta">>>> </span>plt.show()<!-- HTML_TAG_END --></pre></div> <div class="flex justify-center" data-svelte-h="svelte-nsecok"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/semantic-seg-preds.png" alt="Image of bedroom overlaid with segmentation map"></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/tasks/semantic_segmentation.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_z647wz = {
	assets: "/docs/transformers/pr_33913/en",
	base: "/docs/transformers/pr_33913/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/transformers/pr_33913/en/_app/immutable/entry/start.b67f883f.js"),
	import("/docs/transformers/pr_33913/en/_app/immutable/entry/app.e436b1f2.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 434],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 165 kB
Xet hash:: 9fb3b04478d7dde8296139f15dc397c6c64e3fc4299987861a2732a6b3bcfb89

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.