Buckets:

hf-doc-build
/

doc-dev

Files
xet
hf-doc-build/doc-dev / transformers /pr_36708 /en /_app /immutable /nodes /147.CevHeELY.js
HuggingFaceDocBuilder
8 days ago
download
raw
89.5 kB
import"../chunks/DsnmJJEf.js";import{i as fe,h as je,C as Je,H as s,a as n,D as t,E as Ue,s as _e}from"../chunks/D6eQvJIN.js";import{p as we,o as ve,s as e,f as p,a as c,b as Ie,c as l,d as h,r as i,n as m}from"../chunks/D9TCMYxZ.js";import{T as ke}from"../chunks/eoUhp3Ag.js";import{E as j}from"../chunks/D8azpo8f.js";const Be='{"title":"EdgeTAM","local":"edgetam","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[{"title":"Automatic Mask Generation with Pipeline","local":"automatic-mask-generation-with-pipeline","sections":[],"depth":3},{"title":"Basic Image Segmentation","local":"basic-image-segmentation","sections":[{"title":"Single Point Click","local":"single-point-click","sections":[],"depth":4},{"title":"Multiple Points for Refinement","local":"multiple-points-for-refinement","sections":[],"depth":4},{"title":"Bounding Box Input","local":"bounding-box-input","sections":[],"depth":4},{"title":"Multiple Objects Segmentation","local":"multiple-objects-segmentation","sections":[],"depth":4}],"depth":3},{"title":"Batch Inference","local":"batch-inference","sections":[{"title":"Batched Images","local":"batched-images","sections":[],"depth":4},{"title":"Batched Objects per Image","local":"batched-objects-per-image","sections":[],"depth":4},{"title":"Batched Images with Batched Objects and Multiple Points","local":"batched-images-with-batched-objects-and-multiple-points","sections":[],"depth":4},{"title":"Batched Bounding Boxes","local":"batched-bounding-boxes","sections":[],"depth":4}],"depth":3},{"title":"Using Previous Masks as Input","local":"using-previous-masks-as-input","sections":[],"depth":3}],"depth":2},{"title":"EdgeTamConfig","local":"transformers.EdgeTamConfig","sections":[],"depth":2},{"title":"EdgeTamVisionConfig","local":"transformers.EdgeTamVisionConfig","sections":[],"depth":2},{"title":"EdgeTamMaskDecoderConfig","local":"transformers.EdgeTamMaskDecoderConfig","sections":[],"depth":2},{"title":"EdgeTamPromptEncoderConfig","local":"transformers.EdgeTamPromptEncoderConfig","sections":[],"depth":2},{"title":"EdgeTamVisionModel","local":"transformers.EdgeTamVisionModel","sections":[],"depth":2},{"title":"EdgeTamModel","local":"transformers.EdgeTamModel","sections":[],"depth":2}],"depth":1}';var Ne=h('<meta name="hf:doc:metadata"/>'),J=h("<p>Example:</p> <!>",1),Ee=h(`<p>Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code> instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.</p>`),Ce=h(`<p></p> <p><em>This model was published in HF papers on 2025-01-13 and contributed to Hugging Face Transformers on 2025-09-29.</em></p> <!> <div style="float: right;"><div class="flex flex-wrap space-x-1"><img alt="SDPA" src="https://img.shields.io/badge/SDPA-DE3412?style=flat&amp;logo=pytorch&amp;logoColor=white"/> <img alt="FlashAttention" src="https://img.shields.io/badge/%E2%9A%A1%EF%B8%8E%20FlashAttention-eae0c8?style=flat"/></div></div> <!> <!> <p>The EdgeTAM model was proposed in <a href="https://huggingface.co/papers/2501.07256" rel="nofollow">EdgeTAM: On-Device Track Anything Model</a> Chong Zhou, Chenchen Zhu, Yunyang Xiong, Saksham Suri, Fanyi Xiao, Lemeng Wu, Raghuraman Krishnamoorthi, Bo Dai, Chen Change Loy, Vikas Chandra, Bilge Soran.</p> <p>EdgeTAM is an efficient adaptation of SAM 2 that introduces a 2D Spatial Perceiver architecture to optimize memory attention mechanisms for real-time video segmentation on mobile devices.</p> <p>The abstract from the paper is the following:</p> <p><em>On top of Segment Anything Model (SAM), SAM 2 further extends its capability from image to video inputs through a memory bank mechanism and obtains a remarkable performance compared with previous methods, making it a foundation model for video segmentation task. In this paper, we aim at making SAM 2 much more efficient so that it even runs on mobile devices while maintaining a comparable performance. Despite several works optimizing SAM for better efficiency, we find they are not sufficient for SAM 2 because they all focus on compressing the image encoder, while our benchmark shows that the newly introduced memory attention blocks are also the latency bottleneck. Given this observation, we propose EdgeTAM, which leverages a novel 2D Spatial Perceiver to reduce the computational cost. In particular, the proposed 2D Spatial Perceiver encodes the densely stored frame-level memories with a lightweight Transformer that contains a fixed set of learnable queries. Given that video segmentation is a dense prediction task, we find preserving the spatial structure of the memories is essential so that the queries are split into global-level and patch-level groups. We also propose a distillation pipeline that further improves the performance without inference overhead. As a result, EdgeTAM achieves 87.7, 70.0, 72.3, and 71.7 J&F on DAVIS 2017, MOSE, SA-V val, and SA-V test, while running at 16 FPS on iPhone 15 Pro Max.</em></p> <p>This model was contributed by <a href="https://huggingface.co/yonigozlan" rel="nofollow">yonigozlan</a>.
The original code can be found <a href="https://github.com/facebookresearch/EdgeTAM" rel="nofollow">here</a>.</p> <!> <!> <p>EdgeTAM can be used for automatic mask generation to segment all objects in an image using the <code>mask-generation</code> pipeline:</p> <!> <!> <!> <p>You can segment objects by providing a single point click on the object you want to segment:</p> <!> <!> <p>You can provide multiple points to refine the segmentation:</p> <!> <!> <p>EdgeTAM also supports bounding box inputs for segmentation:</p> <!> <!> <p>You can segment multiple objects simultaneously:</p> <!> <!> <!> <p>Process multiple images simultaneously for improved efficiency:</p> <!> <!> <p>Segment multiple objects within each image using batch inference:</p> <!> <!> <p>Handle complex batch scenarios with multiple points per object:</p> <!> <!> <p>Process multiple images with bounding box inputs:</p> <!> <!> <p>EdgeTAM can use masks from previous predictions as input to refine segmentation:</p> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This is the configuration class to store the configuration of a EdgeTamModel. It is used to instantiate a Edgetam
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the <a href="https://huggingface.co/yonigozlan/EdgeTAM-hf" rel="nofollow">yonigozlan/EdgeTAM-hf</a></p> <p>Configuration objects inherit from <a href="/docs/transformers/pr_36708/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_36708/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> for more information.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This is the configuration class to store the configuration of a EdgeTamModel. It is used to instantiate a Edgetam
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the <a href="https://huggingface.co/yonigozlan/EdgeTAM-hf" rel="nofollow">yonigozlan/EdgeTAM-hf</a></p> <p>Configuration objects inherit from <a href="/docs/transformers/pr_36708/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_36708/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> for more information.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This is the configuration class to store the configuration of a EdgeTamModel. It is used to instantiate a Edgetam
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the <a href="https://huggingface.co/yonigozlan/EdgeTAM-hf" rel="nofollow">yonigozlan/EdgeTAM-hf</a></p> <p>Configuration objects inherit from <a href="/docs/transformers/pr_36708/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_36708/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> for more information.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This is the configuration class to store the configuration of a EdgeTamModel. It is used to instantiate a Edgetam
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the <a href="https://huggingface.co/yonigozlan/EdgeTAM-hf" rel="nofollow">yonigozlan/EdgeTAM-hf</a></p> <p>Configuration objects inherit from <a href="/docs/transformers/pr_36708/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/pr_36708/en/main_classes/configuration#transformers.PreTrainedConfig">PreTrainedConfig</a> for more information.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The vision model from EdgeTAM without any head or projection on top.</p> <p>This model inherits from <a href="/docs/transformers/pr_36708/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Segment Anything Model 2 (SAM 2) for generating segmentation masks, given an input image and
input points and labels, boxes, or masks.</p> <p>This model inherits from <a href="/docs/transformers/pr_36708/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)</p> <p>This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>The <a href="/docs/transformers/pr_36708/en/model_doc/edgetam#transformers.EdgeTamModel">EdgeTamModel</a> forward method, overrides the <code>__call__</code> special method.</p> <!> <ul><li><strong>iou_scores</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, point_batch_size, num_masks)</code>) — The Intersection over Union (IoU) scores of the predicted masks.</li> <li><strong>pred_masks</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, point_batch_size, num_masks, height, width)</code>) — The predicted low-resolution masks. This is an alias for <code>low_res_masks</code>. These masks need to be post-processed
by the processor to be brought to the original image size.</li> <li><strong>object_score_logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, point_batch_size, 1)</code>) — Logits for the object score, indicating if an object is present.</li> <li><strong>image_embeddings</strong> (<code>tuple(torch.FloatTensor)</code>) — The features from the FPN, which are used by the mask decoder. This is a tuple of <code>torch.FloatTensor</code> where each
tensor has shape <code>(batch_size, channels, height, width)</code>.</li> <li><strong>vision_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of each stage) of shape <code>(batch_size, height, width, hidden_size)</code>.
Hidden-states of the vision model at the output of each stage.</li> <li><strong>vision_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.
Attentions weights of the vision model.</li> <li><strong>mask_decoder_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.
Attentions weights of the mask decoder.</li></ul> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <ul><li><p><strong>last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, height, width, hidden_size)</code>) — Sequence of hidden-states at the output of the last layer of the model.</p></li> <li><p><strong>pooler_output</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, hidden_size)</code>) — Last layer hidden-state of the first token of the sequence (classification token) after further processing
through the layers used for the auxiliary pretraining task. E.g. for BERT-family of models, this returns
the classification token after processing through a linear layer and a tanh activation function. The linear
layer weights are trained from the next sentence prediction (classification) objective during pretraining.</p></li> <li><p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p> <p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p></li> <li><p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p> <p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p></li> <li><p><strong>fpn_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>) — Tuple of <code>torch.FloatTensor</code> (one for each feature level, from high to low resolution) of shape <code>(batch_size, hidden_size, height, width)</code>. Feature maps from the Feature Pyramid Network neck.</p></li> <li><p><strong>fpn_position_encoding</strong> (<code>tuple(torch.FloatTensor)</code>) — Tuple of <code>torch.FloatTensor</code> (one for each feature level, from high to low resolution) of shape <code>(batch_size, hidden_size, height, width)</code>. Positional encodings corresponding to the <code>fpn_hidden_states</code>.</p></li></ul> <!></div></div> <!> <p></p>`,1);function Re(de,pe){we(pe,!1),ve(()=>{new URLSearchParams(window.location.search).get("fw")}),fe();var U=Ce();je("ojtyd",a=>{var r=Ne();_e(r,"content",Be),c(a,r)});var _=e(p(U),4);Je(_,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var w=e(_,4);s(w,{title:"EdgeTAM",local:"edgetam",headingTag:"h1"});var v=e(w,2);s(v,{title:"Overview",local:"overview",headingTag:"h2"});var I=e(v,12);s(I,{title:"Usage example",local:"usage-example",headingTag:"h2"});var k=e(I,2);s(k,{title:"Automatic Mask Generation with Pipeline",local:"automatic-mask-generation-with-pipeline",headingTag:"h3"});var B=e(k,4);n(B,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBJTBBZ2VuZXJhdG9yJTIwJTNEJTIwcGlwZWxpbmUoJTIybWFzay1nZW5lcmF0aW9uJTIyJTJDJTIwbW9kZWwlM0QlMjJ5b25pZ296bGFuJTJGZWRnZXRhbS0xJTIyJTJDJTIwZGV2aWNlJTNEMCklMEFpbWFnZV91cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZoZi1pbnRlcm5hbC10ZXN0aW5nJTJGc2FtMi1maXh0dXJlcyUyRnJlc29sdmUlMkZtYWluJTJGdHJ1Y2suanBnJTIyJTBBb3V0cHV0cyUyMCUzRCUyMGdlbmVyYXRvcihpbWFnZV91cmwlMkMlMjBwb2ludHNfcGVyX2JhdGNoJTNENjQpJTBBJTBBbGVuKG91dHB1dHMlNUIlMjJtYXNrcyUyMiU1RCklMjAlMjAlMjMlMjBOdW1iZXIlMjBvZiUyMG1hc2tzJTIwZ2VuZXJhdGVkJTBBMzk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
generator = pipeline(<span class="hljs-string">&quot;mask-generation&quot;</span>, model=<span class="hljs-string">&quot;yonigozlan/edgetam-1&quot;</span>, device=<span class="hljs-number">0</span>)
image_url = <span class="hljs-string">&quot;https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg&quot;</span>
outputs = generator(image_url, points_per_batch=<span class="hljs-number">64</span>)
<span class="hljs-built_in">len</span>(outputs[<span class="hljs-string">&quot;masks&quot;</span>])  <span class="hljs-comment"># Number of masks generated</span>
<span class="hljs-number">39</span>`,lang:"python",wrap:!1});var N=e(B,2);s(N,{title:"Basic Image Segmentation",local:"basic-image-segmentation",headingTag:"h3"});var E=e(N,2);s(E,{title:"Single Point Click",local:"single-point-click",headingTag:"h4"});var C=e(E,4);n(C,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNhbTJQcm9jZXNzb3IlMkMlMjBFZGdlVGFtTW9kZWwlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjBQSUwlMjBpbXBvcnQlMjBJbWFnZSUwQWltcG9ydCUyMHJlcXVlc3RzJTBBJTBBJTBBbW9kZWwlMjAlM0QlMjBFZGdlVGFtTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMnlvbmlnb3psYW4lMkZlZGdldGFtLTElMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBTYW0yUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJ5b25pZ296bGFuJTJGZWRnZXRhbS0xJTIyKSUwQSUwQWltYWdlX3VybCUyMCUzRCUyMCUyMmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZkYXRhc2V0cyUyRmhmLWludGVybmFsLXRlc3RpbmclMkZzYW0yLWZpeHR1cmVzJTJGcmVzb2x2ZSUyRm1haW4lMkZ0cnVjay5qcGclMjIlMEFyYXdfaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldChpbWFnZV91cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpLmNvbnZlcnQoJTIyUkdCJTIyKSUwQSUwQWlucHV0X3BvaW50cyUyMCUzRCUyMCU1QiU1QiU1QiU1QjUwMCUyQyUyMDM3NSU1RCU1RCU1RCU1RCUyMCUyMCUyMyUyMFNpbmdsZSUyMHBvaW50JTIwY2xpY2slMkMlMjA0JTIwZGltZW5zaW9ucyUyMChpbWFnZV9kaW0lMkMlMjBvYmplY3RfZGltJTJDJTIwcG9pbnRfcGVyX29iamVjdF9kaW0lMkMlMjBjb29yZGluYXRlcyklMEFpbnB1dF9sYWJlbHMlMjAlM0QlMjAlNUIlNUIlNUIxJTVEJTVEJTVEJTIwJTIwJTIzJTIwMSUyMGZvciUyMHBvc2l0aXZlJTIwY2xpY2slMkMlMjAwJTIwZm9yJTIwbmVnYXRpdmUlMjBjbGljayUyQyUyMDMlMjBkaW1lbnNpb25zJTIwKGltYWdlX2RpbSUyQyUyMG9iamVjdF9kaW0lMkMlMjBwb2ludF9sYWJlbCklMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoaW1hZ2VzJTNEcmF3X2ltYWdlJTJDJTIwaW5wdXRfcG9pbnRzJTNEaW5wdXRfcG9pbnRzJTJDJTIwaW5wdXRfbGFiZWxzJTNEaW5wdXRfbGFiZWxzJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpJTBBJTBBbWFza3MlMjAlM0QlMjBwcm9jZXNzb3IucG9zdF9wcm9jZXNzX21hc2tzKG91dHB1dHMucHJlZF9tYXNrcy5jcHUoKSUyQyUyMGlucHV0cyU1QiUyMm9yaWdpbmFsX3NpemVzJTIyJTVEKSU1QjAlNUQlMEElMEElMjMlMjBUaGUlMjBtb2RlbCUyMG91dHB1dHMlMjBtdWx0aXBsZSUyMG1hc2slMjBwcmVkaWN0aW9ucyUyMHJhbmtlZCUyMGJ5JTIwcXVhbGl0eSUyMHNjb3JlJTBBcHJpbnQoZiUyMkdlbmVyYXRlZCUyMCU3Qm1hc2tzLnNoYXBlJTVCMSU1RCU3RCUyMG1hc2tzJTIwd2l0aCUyMHNoYXBlJTIwJTdCbWFza3Muc2hhcGUlN0QlMjIpJTBBR2VuZXJhdGVkJTIwMyUyMG1hc2tzJTIwd2l0aCUyMHNoYXBlJTIwdG9yY2guU2l6ZSglNUIxJTJDJTIwMyUyQyUyMDEyMDAlMkMlMjAxODAwJTVEKSUwQXByaW50KGYlMjJJb1UlMjBzY29yZXMlM0ElMjAlN0JvdXRwdXRzLmlvdV9zY29yZXMuc3F1ZWV6ZSgpJTdEJTIyKSUwQUlvVSUyMHNjb3JlcyUzQSUyMHRlbnNvciglNUIwLjA0NjMlMkMlMjAwLjQ4NTklMkMlMjAwLjc2MTYlNUQlMkMlMjBkZXZpY2UlM0QnY3VkYSUzQTAnKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Sam2Processor, EdgeTamModel
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">import</span> requests
model = EdgeTamModel.from_pretrained(<span class="hljs-string">&quot;yonigozlan/edgetam-1&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
processor = Sam2Processor.from_pretrained(<span class="hljs-string">&quot;yonigozlan/edgetam-1&quot;</span>)
image_url = <span class="hljs-string">&quot;https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg&quot;</span>
raw_image = Image.<span class="hljs-built_in">open</span>(requests.get(image_url, stream=<span class="hljs-literal">True</span>).raw).convert(<span class="hljs-string">&quot;RGB&quot;</span>)
input_points = [[[[<span class="hljs-number">500</span>, <span class="hljs-number">375</span>]]]]  <span class="hljs-comment"># Single point click, 4 dimensions (image_dim, object_dim, point_per_object_dim, coordinates)</span>
input_labels = [[[<span class="hljs-number">1</span>]]]  <span class="hljs-comment"># 1 for positive click, 0 for negative click, 3 dimensions (image_dim, object_dim, point_label)</span>
inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs)
masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>])[<span class="hljs-number">0</span>]
<span class="hljs-comment"># The model outputs multiple mask predictions ranked by quality score</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Generated <span class="hljs-subst">{masks.shape[<span class="hljs-number">1</span>]}</span> masks with shape <span class="hljs-subst">{masks.shape}</span>&quot;</span>)
Generated <span class="hljs-number">3</span> masks <span class="hljs-keyword">with</span> shape torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">3</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">1800</span>])
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;IoU scores: <span class="hljs-subst">{outputs.iou_scores.squeeze()}</span>&quot;</span>)
IoU scores: tensor([<span class="hljs-number">0.0463</span>, <span class="hljs-number">0.4859</span>, <span class="hljs-number">0.7616</span>], device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)`,lang:"python",wrap:!1});var V=e(C,2);s(V,{title:"Multiple Points for Refinement",local:"multiple-points-for-refinement",headingTag:"h4"});var W=e(V,4);n(W,{code:"JTIzJTIwQWRkJTIwYm90aCUyMHBvc2l0aXZlJTIwYW5kJTIwbmVnYXRpdmUlMjBwb2ludHMlMjB0byUyMHJlZmluZSUyMHRoZSUyMG1hc2slMEFpbnB1dF9wb2ludHMlMjAlM0QlMjAlNUIlNUIlNUIlNUI1MDAlMkMlMjAzNzUlNUQlMkMlMjAlNUIxMTI1JTJDJTIwNjI1JTVEJTVEJTVEJTVEJTIwJTIwJTIzJTIwTXVsdGlwbGUlMjBwb2ludHMlMjBmb3IlMjByZWZpbmVtZW50JTBBaW5wdXRfbGFiZWxzJTIwJTNEJTIwJTVCJTVCJTVCMSUyQyUyMDElNUQlNUQlNUQlMjAlMjAlMjMlMjBCb3RoJTIwcG9zaXRpdmUlMjBjbGlja3MlMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoaW1hZ2VzJTNEcmF3X2ltYWdlJTJDJTIwaW5wdXRfcG9pbnRzJTNEaW5wdXRfcG9pbnRzJTJDJTIwaW5wdXRfbGFiZWxzJTNEaW5wdXRfbGFiZWxzJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpJTBBJTBBbWFza3MlMjAlM0QlMjBwcm9jZXNzb3IucG9zdF9wcm9jZXNzX21hc2tzKG91dHB1dHMucHJlZF9tYXNrcy5jcHUoKSUyQyUyMGlucHV0cyU1QiUyMm9yaWdpbmFsX3NpemVzJTIyJTVEKSU1QjAlNUQlMEFwcmludChmJTIySW9VJTIwc2NvcmVzJTNBJTIwJTdCb3V0cHV0cy5pb3Vfc2NvcmVzLnNxdWVlemUoKSU3RCUyMiklMEFJb1UlMjBzY29yZXMlM0ElMjB0ZW5zb3IoJTVCMC44MzYyJTJDJTIwMC42OTAwJTJDJTIwMC4yMTIwJTVEJTJDJTIwZGV2aWNlJTNEJ2N1ZGElM0EwJyk=",highlighted:`<span class="hljs-comment"># Add both positive and negative points to refine the mask</span>
input_points = [[[[<span class="hljs-number">500</span>, <span class="hljs-number">375</span>], [<span class="hljs-number">1125</span>, <span class="hljs-number">625</span>]]]]  <span class="hljs-comment"># Multiple points for refinement</span>
input_labels = [[[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]]  <span class="hljs-comment"># Both positive clicks</span>
inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs)
masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>])[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;IoU scores: <span class="hljs-subst">{outputs.iou_scores.squeeze()}</span>&quot;</span>)
IoU scores: tensor([<span class="hljs-number">0.8362</span>, <span class="hljs-number">0.6900</span>, <span class="hljs-number">0.2120</span>], device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)`,lang:"python",wrap:!1});var Z=e(W,2);s(Z,{title:"Bounding Box Input",local:"bounding-box-input",headingTag:"h4"});var z=e(Z,4);n(z,{code:"JTIzJTIwRGVmaW5lJTIwYm91bmRpbmclMjBib3glMjBhcyUyMCU1QnhfbWluJTJDJTIweV9taW4lMkMlMjB4X21heCUyQyUyMHlfbWF4JTVEJTBBaW5wdXRfYm94ZXMlMjAlM0QlMjAlNUIlNUIlNUI3NSUyQyUyMDI3NSUyQyUyMDE3MjUlMkMlMjA4NTAlNUQlNUQlNUQlMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoaW1hZ2VzJTNEcmF3X2ltYWdlJTJDJTIwaW5wdXRfYm94ZXMlM0RpbnB1dF9ib3hlcyUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKG1vZGVsLmRldmljZSklMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQSUwQW1hc2tzJTIwJTNEJTIwcHJvY2Vzc29yLnBvc3RfcHJvY2Vzc19tYXNrcyhvdXRwdXRzLnByZWRfbWFza3MuY3B1KCklMkMlMjBpbnB1dHMlNUIlMjJvcmlnaW5hbF9zaXplcyUyMiU1RCklNUIwJTVEJTBBcHJpbnQoZiUyMklvVSUyMHNjb3JlcyUzQSUyMCU3Qm91dHB1dHMuaW91X3Njb3Jlcy5zcXVlZXplKCklN0QlMjIpJTBBSW9VJTIwc2NvcmVzJTNBJTIwdGVuc29yKCU1QjAuOTMwMSUyQyUyMDAuOTM0OCUyQyUyMDAuNjYwNSU1RCUyQyUyMGRldmljZSUzRCdjdWRhJTNBMCcp",highlighted:`<span class="hljs-comment"># Define bounding box as [x_min, y_min, x_max, y_max]</span>
input_boxes = [[[<span class="hljs-number">75</span>, <span class="hljs-number">275</span>, <span class="hljs-number">1725</span>, <span class="hljs-number">850</span>]]]
inputs = processor(images=raw_image, input_boxes=input_boxes, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs)
masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>])[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;IoU scores: <span class="hljs-subst">{outputs.iou_scores.squeeze()}</span>&quot;</span>)
IoU scores: tensor([<span class="hljs-number">0.9301</span>, <span class="hljs-number">0.9348</span>, <span class="hljs-number">0.6605</span>], device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)`,lang:"python",wrap:!1});var G=e(z,2);s(G,{title:"Multiple Objects Segmentation",local:"multiple-objects-segmentation",headingTag:"h4"});var R=e(G,4);n(R,{code:"JTIzJTIwRGVmaW5lJTIwcG9pbnRzJTIwZm9yJTIwdHdvJTIwZGlmZmVyZW50JTIwb2JqZWN0cyUwQWlucHV0X3BvaW50cyUyMCUzRCUyMCU1QiU1QiU1QiU1QjUwMCUyQyUyMDM3NSU1RCU1RCUyQyUyMCU1QiU1QjY1MCUyQyUyMDc1MCU1RCU1RCU1RCU1RCUyMCUyMCUyMyUyMFBvaW50cyUyMGZvciUyMHR3byUyMG9iamVjdHMlMjBpbiUyMHNhbWUlMjBpbWFnZSUwQWlucHV0X2xhYmVscyUyMCUzRCUyMCU1QiU1QiU1QjElNUQlMkMlMjAlNUIxJTVEJTVEJTVEJTIwJTIwJTIzJTIwUG9zaXRpdmUlMjBjbGlja3MlMjBmb3IlMjBib3RoJTIwb2JqZWN0cyUwQSUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3NvcihpbWFnZXMlM0RyYXdfaW1hZ2UlMkMlMjBpbnB1dF9wb2ludHMlM0RpbnB1dF9wb2ludHMlMkMlMjBpbnB1dF9sYWJlbHMlM0RpbnB1dF9sYWJlbHMlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyUyQyUyMG11bHRpbWFza19vdXRwdXQlM0RGYWxzZSklMEElMEElMjMlMjBFYWNoJTIwb2JqZWN0JTIwZ2V0cyUyMGl0cyUyMG93biUyMG1hc2slMEFtYXNrcyUyMCUzRCUyMHByb2Nlc3Nvci5wb3N0X3Byb2Nlc3NfbWFza3Mob3V0cHV0cy5wcmVkX21hc2tzLmNwdSgpJTJDJTIwaW5wdXRzJTVCJTIyb3JpZ2luYWxfc2l6ZXMlMjIlNUQpJTVCMCU1RCUwQXByaW50KGYlMjJHZW5lcmF0ZWQlMjBtYXNrcyUyMGZvciUyMCU3Qm1hc2tzLnNoYXBlJTVCMCU1RCU3RCUyMG9iamVjdHMlMjIpJTBBR2VuZXJhdGVkJTIwbWFza3MlMjBmb3IlMjAyJTIwb2JqZWN0cyUwQXByaW50KGYlMjJJb1UlMjBzY29yZXMlM0ElMjAlN0JvdXRwdXRzLmlvdV9zY29yZXMuc3F1ZWV6ZSgpJTdEJTIyKSUwQUlvVSUyMHNjb3JlcyUzQSUyMHRlbnNvciglNUIwLjc2MTYlMkMlMjAwLjk0NjUlNUQlMkMlMjBkZXZpY2UlM0QnY3VkYSUzQTAnKQ==",highlighted:`<span class="hljs-comment"># Define points for two different objects</span>
input_points = [[[[<span class="hljs-number">500</span>, <span class="hljs-number">375</span>]], [[<span class="hljs-number">650</span>, <span class="hljs-number">750</span>]]]]  <span class="hljs-comment"># Points for two objects in same image</span>
input_labels = [[[<span class="hljs-number">1</span>], [<span class="hljs-number">1</span>]]]  <span class="hljs-comment"># Positive clicks for both objects</span>
inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs, multimask_output=<span class="hljs-literal">False</span>)
<span class="hljs-comment"># Each object gets its own mask</span>
masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>])[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Generated masks for <span class="hljs-subst">{masks.shape[<span class="hljs-number">0</span>]}</span> objects&quot;</span>)
Generated masks <span class="hljs-keyword">for</span> <span class="hljs-number">2</span> objects
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;IoU scores: <span class="hljs-subst">{outputs.iou_scores.squeeze()}</span>&quot;</span>)
IoU scores: tensor([<span class="hljs-number">0.7616</span>, <span class="hljs-number">0.9465</span>], device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)`,lang:"python",wrap:!1});var X=e(R,2);s(X,{title:"Batch Inference",local:"batch-inference",headingTag:"h3"});var Q=e(X,2);s(Q,{title:"Batched Images",local:"batched-images",headingTag:"h4"});var F=e(Q,4);n(F,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNhbTJQcm9jZXNzb3IlMkMlMjBFZGdlVGFtTW9kZWwlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjBQSUwlMjBpbXBvcnQlMjBJbWFnZSUwQWltcG9ydCUyMHJlcXVlc3RzJTBBJTBBJTBBbW9kZWwlMjAlM0QlMjBFZGdlVGFtTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMnlvbmlnb3psYW4lMkZlZGdldGFtLTElMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBTYW0yUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJ5b25pZ296bGFuJTJGZWRnZXRhbS0xJTIyKSUwQSUwQSUyMyUyMExvYWQlMjBtdWx0aXBsZSUyMGltYWdlcyUwQWltYWdlX3VybHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZoZi1pbnRlcm5hbC10ZXN0aW5nJTJGc2FtMi1maXh0dXJlcyUyRnJlc29sdmUlMkZtYWluJTJGdHJ1Y2suanBnJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaGYtaW50ZXJuYWwtdGVzdGluZyUyRnRyYW5zZm9ybWVycy1zeW50aGV0aWMtYXNzZXRzJTJGcmVzb2x2ZSUyRm1haW4lMkZpbWFnZXMlMkZkb2dfc2FtLnBuZyUyMiUwQSU1RCUwQXJhd19pbWFnZXMlMjAlM0QlMjAlNUJJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpLmNvbnZlcnQoJTIyUkdCJTIyKSUyMGZvciUyMHVybCUyMGluJTIwaW1hZ2VfdXJscyU1RCUwQSUwQSUyMyUyMFNpbmdsZSUyMHBvaW50JTIwcGVyJTIwaW1hZ2UlMEFpbnB1dF9wb2ludHMlMjAlM0QlMjAlNUIlNUIlNUIlNUI1MDAlMkMlMjAzNzUlNUQlNUQlNUQlMkMlMjAlNUIlNUIlNUI3NzAlMkMlMjAyMDAlNUQlNUQlNUQlNUQlMjAlMjAlMjMlMjBPbmUlMjBwb2ludCUyMGZvciUyMGVhY2glMjBpbWFnZSUwQWlucHV0X2xhYmVscyUyMCUzRCUyMCU1QiU1QiU1QjElNUQlNUQlMkMlMjAlNUIlNUIxJTVEJTVEJTVEJTIwJTIwJTIzJTIwUG9zaXRpdmUlMjBjbGlja3MlMjBmb3IlMjBib3RoJTIwaW1hZ2VzJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRHJhd19pbWFnZXMlMkMlMjBpbnB1dF9wb2ludHMlM0RpbnB1dF9wb2ludHMlMkMlMjBpbnB1dF9sYWJlbHMlM0RpbnB1dF9sYWJlbHMlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyUyQyUyMG11bHRpbWFza19vdXRwdXQlM0RGYWxzZSklMEElMEElMjMlMjBQb3N0LXByb2Nlc3MlMjBtYXNrcyUyMGZvciUyMGVhY2glMjBpbWFnZSUwQWFsbF9tYXNrcyUyMCUzRCUyMHByb2Nlc3Nvci5wb3N0X3Byb2Nlc3NfbWFza3Mob3V0cHV0cy5wcmVkX21hc2tzLmNwdSgpJTJDJTIwaW5wdXRzJTVCJTIyb3JpZ2luYWxfc2l6ZXMlMjIlNUQpJTBBcHJpbnQoZiUyMlByb2Nlc3NlZCUyMCU3QmxlbihhbGxfbWFza3MpJTdEJTIwaW1hZ2VzJTJDJTIwZWFjaCUyMHdpdGglMjAlN0JhbGxfbWFza3MlNUIwJTVELnNoYXBlJTVCMCU1RCU3RCUyMG9iamVjdHMlMjIpJTBBUHJvY2Vzc2VkJTIwMiUyMGltYWdlcyUyQyUyMGVhY2glMjB3aXRoJTIwMSUyMG9iamVjdHMlMEFwcmludChmJTIySW9VJTIwc2NvcmVzJTNBJTIwJTdCb3V0cHV0cy5pb3Vfc2NvcmVzLnNxdWVlemUoKSU3RCUyMiklMEFJb1UlMjBzY29yZXMlM0ElMjB0ZW5zb3IoJTVCMC43NjE4JTJDJTIwMC43OTk5JTVEJTJDJTIwZGV2aWNlJTNEJ2N1ZGElM0EwJyk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Sam2Processor, EdgeTamModel
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">import</span> requests
model = EdgeTamModel.from_pretrained(<span class="hljs-string">&quot;yonigozlan/edgetam-1&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
processor = Sam2Processor.from_pretrained(<span class="hljs-string">&quot;yonigozlan/edgetam-1&quot;</span>)
<span class="hljs-comment"># Load multiple images</span>
image_urls = [
    <span class="hljs-string">&quot;https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/truck.jpg&quot;</span>,
    <span class="hljs-string">&quot;https://huggingface.co/datasets/hf-internal-testing/transformers-synthetic-assets/resolve/main/images/dog_sam.png&quot;</span>
]
raw_images = [Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw).convert(<span class="hljs-string">&quot;RGB&quot;</span>) <span class="hljs-keyword">for</span> url <span class="hljs-keyword">in</span> image_urls]
<span class="hljs-comment"># Single point per image</span>
input_points = [[[[<span class="hljs-number">500</span>, <span class="hljs-number">375</span>]]], [[[<span class="hljs-number">770</span>, <span class="hljs-number">200</span>]]]]  <span class="hljs-comment"># One point for each image</span>
input_labels = [[[<span class="hljs-number">1</span>]], [[<span class="hljs-number">1</span>]]]  <span class="hljs-comment"># Positive clicks for both images</span>
inputs = processor(images=raw_images, input_points=input_points, input_labels=input_labels, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs, multimask_output=<span class="hljs-literal">False</span>)
<span class="hljs-comment"># Post-process masks for each image</span>
all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>])
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Processed <span class="hljs-subst">{<span class="hljs-built_in">len</span>(all_masks)}</span> images, each with <span class="hljs-subst">{all_masks[<span class="hljs-number">0</span>].shape[<span class="hljs-number">0</span>]}</span> objects&quot;</span>)
Processed <span class="hljs-number">2</span> images, each <span class="hljs-keyword">with</span> <span class="hljs-number">1</span> objects
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;IoU scores: <span class="hljs-subst">{outputs.iou_scores.squeeze()}</span>&quot;</span>)
IoU scores: tensor([<span class="hljs-number">0.7618</span>, <span class="hljs-number">0.7999</span>], device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)`,lang:"python",wrap:!1});var A=e(F,2);s(A,{title:"Batched Objects per Image",local:"batched-objects-per-image",headingTag:"h4"});var Y=e(A,4);n(Y,{code:"JTIzJTIwTXVsdGlwbGUlMjBvYmplY3RzJTIwcGVyJTIwaW1hZ2UlMjAtJTIwZGlmZmVyZW50JTIwbnVtYmVycyUyMG9mJTIwb2JqZWN0cyUyMHBlciUyMGltYWdlJTBBaW5wdXRfcG9pbnRzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTVCJTVCJTVCNTAwJTJDJTIwMzc1JTVEJTVEJTJDJTIwJTVCJTVCNjUwJTJDJTIwNzUwJTVEJTVEJTVEJTJDJTIwJTIwJTIzJTIwVHJ1Y2slMjBpbWFnZSUzQSUyMDIlMjBvYmplY3RzJTBBJTIwJTIwJTIwJTIwJTVCJTVCJTVCNzcwJTJDJTIwMjAwJTVEJTVEJTVEJTIwJTIwJTIzJTIwRG9nJTIwaW1hZ2UlM0ElMjAxJTIwb2JqZWN0JTBBJTVEJTBBaW5wdXRfbGFiZWxzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTVCJTVCMSU1RCUyQyUyMCU1QjElNUQlNUQlMkMlMjAlMjAlMjMlMjBUcnVjayUyMGltYWdlJTNBJTIwcG9zaXRpdmUlMjBjbGlja3MlMjBmb3IlMjBib3RoJTIwb2JqZWN0cyUwQSUyMCUyMCUyMCUyMCU1QiU1QjElNUQlNUQlMjAlMjAlMjMlMjBEb2clMjBpbWFnZSUzQSUyMHBvc2l0aXZlJTIwY2xpY2slMjBmb3IlMjB0aGUlMjBvYmplY3QlMEElNUQlMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoaW1hZ2VzJTNEcmF3X2ltYWdlcyUyQyUyMGlucHV0X3BvaW50cyUzRGlucHV0X3BvaW50cyUyQyUyMGlucHV0X2xhYmVscyUzRGlucHV0X2xhYmVscyUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKG1vZGVsLmRldmljZSklMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzJTJDJTIwbXVsdGltYXNrX291dHB1dCUzREZhbHNlKSUwQSUwQWFsbF9tYXNrcyUyMCUzRCUyMHByb2Nlc3Nvci5wb3N0X3Byb2Nlc3NfbWFza3Mob3V0cHV0cy5wcmVkX21hc2tzLmNwdSgpJTJDJTIwaW5wdXRzJTVCJTIyb3JpZ2luYWxfc2l6ZXMlMjIlNUQp",highlighted:`<span class="hljs-comment"># Multiple objects per image - different numbers of objects per image</span>
input_points = [
    [[[<span class="hljs-number">500</span>, <span class="hljs-number">375</span>]], [[<span class="hljs-number">650</span>, <span class="hljs-number">750</span>]]],  <span class="hljs-comment"># Truck image: 2 objects</span>
    [[[<span class="hljs-number">770</span>, <span class="hljs-number">200</span>]]]  <span class="hljs-comment"># Dog image: 1 object</span>
]
input_labels = [
    [[<span class="hljs-number">1</span>], [<span class="hljs-number">1</span>]],  <span class="hljs-comment"># Truck image: positive clicks for both objects</span>
    [[<span class="hljs-number">1</span>]]  <span class="hljs-comment"># Dog image: positive click for the object</span>
]
inputs = processor(images=raw_images, input_points=input_points, input_labels=input_labels, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs, multimask_output=<span class="hljs-literal">False</span>)
all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>])`,lang:"python",wrap:!1});var x=e(Y,2);s(x,{title:"Batched Images with Batched Objects and Multiple Points",local:"batched-images-with-batched-objects-and-multiple-points",headingTag:"h4"});var H=e(x,4);n(H,{code:"JTIzJTIwQWRkJTIwZ3JvY2VyaWVzJTIwaW1hZ2UlMjBmb3IlMjBtb3JlJTIwY29tcGxleCUyMGV4YW1wbGUlMEFncm9jZXJpZXNfdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaGYtaW50ZXJuYWwtdGVzdGluZyUyRnNhbTItZml4dHVyZXMlMkZyZXNvbHZlJTJGbWFpbiUyRmdyb2Nlcmllcy5qcGclMjIlMEFncm9jZXJpZXNfaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldChncm9jZXJpZXNfdXJsJTJDJTIwc3RyZWFtJTNEVHJ1ZSkucmF3KS5jb252ZXJ0KCUyMlJHQiUyMiklMEFyYXdfaW1hZ2VzJTIwJTNEJTIwJTVCcmF3X2ltYWdlcyU1QjAlNUQlMkMlMjBncm9jZXJpZXNfaW1hZ2UlNUQlMjAlMjAlMjMlMjBVc2UlMjB0cnVjayUyMGFuZCUyMGdyb2NlcmllcyUyMGltYWdlcyUwQSUwQSUyMyUyMENvbXBsZXglMjBiYXRjaGluZyUzQSUyMG11bHRpcGxlJTIwaW1hZ2VzJTJDJTIwbXVsdGlwbGUlMjBvYmplY3RzJTJDJTIwbXVsdGlwbGUlMjBwb2ludHMlMjBwZXIlMjBvYmplY3QlMEFpbnB1dF9wb2ludHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlNUIlNUIlNUI1MDAlMkMlMjAzNzUlNUQlNUQlMkMlMjAlNUIlNUI2NTAlMkMlMjA3NTAlNUQlNUQlNUQlMkMlMjAlMjAlMjMlMjBUcnVjayUyMGltYWdlJTNBJTIwMiUyMG9iamVjdHMlMjB3aXRoJTIwMSUyMHBvaW50JTIwZWFjaCUwQSUyMCUyMCUyMCUyMCU1QiU1QiU1QjQwMCUyQyUyMDMwMCU1RCU1RCUyQyUyMCU1QiU1QjYzMCUyQyUyMDMwMCU1RCUyQyUyMCU1QjU1MCUyQyUyMDMwMCU1RCU1RCU1RCUyMCUyMCUyMyUyMEdyb2NlcmllcyUyMGltYWdlJTNBJTIwb2JqMSUyMGhhcyUyMDElMjBwb2ludCUyQyUyMG9iajIlMjBoYXMlMjAyJTIwcG9pbnRzJTBBJTVEJTBBaW5wdXRfbGFiZWxzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTVCJTVCMSU1RCUyQyUyMCU1QjElNUQlNUQlMkMlMjAlMjAlMjMlMjBUcnVjayUyMGltYWdlJTNBJTIwcG9zaXRpdmUlMjBjbGlja3MlMEElMjAlMjAlMjAlMjAlNUIlNUIxJTVEJTJDJTIwJTVCMSUyQyUyMDElNUQlNUQlMjAlMjAlMjMlMjBHcm9jZXJpZXMlMjBpbWFnZSUzQSUyMHBvc2l0aXZlJTIwY2xpY2tzJTIwZm9yJTIwcmVmaW5lbWVudCUwQSU1RCUwQSUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3NvcihpbWFnZXMlM0RyYXdfaW1hZ2VzJTJDJTIwaW5wdXRfcG9pbnRzJTNEaW5wdXRfcG9pbnRzJTJDJTIwaW5wdXRfbGFiZWxzJTNEaW5wdXRfbGFiZWxzJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMlMkMlMjBtdWx0aW1hc2tfb3V0cHV0JTNERmFsc2UpJTBBJTBBYWxsX21hc2tzJTIwJTNEJTIwcHJvY2Vzc29yLnBvc3RfcHJvY2Vzc19tYXNrcyhvdXRwdXRzLnByZWRfbWFza3MuY3B1KCklMkMlMjBpbnB1dHMlNUIlMjJvcmlnaW5hbF9zaXplcyUyMiU1RCk=",highlighted:`<span class="hljs-comment"># Add groceries image for more complex example</span>
groceries_url = <span class="hljs-string">&quot;https://huggingface.co/datasets/hf-internal-testing/sam2-fixtures/resolve/main/groceries.jpg&quot;</span>
groceries_image = Image.<span class="hljs-built_in">open</span>(requests.get(groceries_url, stream=<span class="hljs-literal">True</span>).raw).convert(<span class="hljs-string">&quot;RGB&quot;</span>)
raw_images = [raw_images[<span class="hljs-number">0</span>], groceries_image]  <span class="hljs-comment"># Use truck and groceries images</span>
<span class="hljs-comment"># Complex batching: multiple images, multiple objects, multiple points per object</span>
input_points = [
    [[[<span class="hljs-number">500</span>, <span class="hljs-number">375</span>]], [[<span class="hljs-number">650</span>, <span class="hljs-number">750</span>]]],  <span class="hljs-comment"># Truck image: 2 objects with 1 point each</span>
    [[[<span class="hljs-number">400</span>, <span class="hljs-number">300</span>]], [[<span class="hljs-number">630</span>, <span class="hljs-number">300</span>], [<span class="hljs-number">550</span>, <span class="hljs-number">300</span>]]]  <span class="hljs-comment"># Groceries image: obj1 has 1 point, obj2 has 2 points</span>
]
input_labels = [
    [[<span class="hljs-number">1</span>], [<span class="hljs-number">1</span>]],  <span class="hljs-comment"># Truck image: positive clicks</span>
    [[<span class="hljs-number">1</span>], [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]  <span class="hljs-comment"># Groceries image: positive clicks for refinement</span>
]
inputs = processor(images=raw_images, input_points=input_points, input_labels=input_labels, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs, multimask_output=<span class="hljs-literal">False</span>)
all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>])`,lang:"python",wrap:!1});var S=e(H,2);s(S,{title:"Batched Bounding Boxes",local:"batched-bounding-boxes",headingTag:"h4"});var D=e(S,4);n(D,{code:"JTIzJTIwTXVsdGlwbGUlMjBib3VuZGluZyUyMGJveGVzJTIwcGVyJTIwaW1hZ2UlMjAodXNpbmclMjB0cnVjayUyMGFuZCUyMGdyb2NlcmllcyUyMGltYWdlcyklMEFpbnB1dF9ib3hlcyUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCU1QiU1Qjc1JTJDJTIwMjc1JTJDJTIwMTcyNSUyQyUyMDg1MCU1RCUyQyUyMCU1QjQyNSUyQyUyMDYwMCUyQyUyMDcwMCUyQyUyMDg3NSU1RCUyQyUyMCU1QjEzNzUlMkMlMjA1NTAlMkMlMjAxNjUwJTJDJTIwODAwJTVEJTJDJTIwJTVCMTI0MCUyQyUyMDY3NSUyQyUyMDE0MDAlMkMlMjA3NTAlNUQlNUQlMkMlMjAlMjAlMjMlMjBUcnVjayUyMGltYWdlJTNBJTIwNCUyMGJveGVzJTBBJTIwJTIwJTIwJTIwJTVCJTVCNDUwJTJDJTIwMTcwJTJDJTIwNTIwJTJDJTIwMzUwJTVEJTJDJTIwJTVCMzUwJTJDJTIwMTkwJTJDJTIwNDUwJTJDJTIwMzUwJTVEJTJDJTIwJTVCNTAwJTJDJTIwMTcwJTJDJTIwNTgwJTJDJTIwMzUwJTVEJTJDJTIwJTVCNTgwJTJDJTIwMTcwJTJDJTIwNjQwJTJDJTIwMzUwJTVEJTVEJTIwJTIwJTIzJTIwR3JvY2VyaWVzJTIwaW1hZ2UlM0ElMjA0JTIwYm94ZXMlMEElNUQlMEElMEElMjMlMjBVcGRhdGUlMjBpbWFnZXMlMjBmb3IlMjB0aGlzJTIwZXhhbXBsZSUwQXJhd19pbWFnZXMlMjAlM0QlMjAlNUJyYXdfaW1hZ2VzJTVCMCU1RCUyQyUyMGdyb2Nlcmllc19pbWFnZSU1RCUyMCUyMCUyMyUyMHRydWNrJTIwYW5kJTIwZ3JvY2VyaWVzJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRHJhd19pbWFnZXMlMkMlMjBpbnB1dF9ib3hlcyUzRGlucHV0X2JveGVzJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMlMkMlMjBtdWx0aW1hc2tfb3V0cHV0JTNERmFsc2UpJTBBJTBBYWxsX21hc2tzJTIwJTNEJTIwcHJvY2Vzc29yLnBvc3RfcHJvY2Vzc19tYXNrcyhvdXRwdXRzLnByZWRfbWFza3MuY3B1KCklMkMlMjBpbnB1dHMlNUIlMjJvcmlnaW5hbF9zaXplcyUyMiU1RCklMEFwcmludChmJTIyUHJvY2Vzc2VkJTIwJTdCbGVuKGlucHV0X2JveGVzKSU3RCUyMGltYWdlcyUyMHdpdGglMjAlN0JsZW4oaW5wdXRfYm94ZXMlNUIwJTVEKSU3RCUyMGFuZCUyMCU3QmxlbihpbnB1dF9ib3hlcyU1QjElNUQpJTdEJTIwYm94ZXMlMjByZXNwZWN0aXZlbHklMjIpJTBBUHJvY2Vzc2VkJTIwMiUyMGltYWdlcyUyMHdpdGglMjA0JTIwYW5kJTIwNCUyMGJveGVzJTIwcmVzcGVjdGl2ZWx5JTBBcHJpbnQoZiUyMklvVSUyMHNjb3JlcyUzQSUyMCU3Qm91dHB1dHMuaW91X3Njb3Jlcy5zcXVlZXplKCklN0QlMjIpJTBBSW9VJTIwc2NvcmVzJTNBJTIwdGVuc29yKCU1QjAuOTMwMSUyQyUyMDAuOTM0OCUyQyUyMDAuNjYwNSUyQyUyMDAuOTQ2NSU1RCUyQyUyMGRldmljZSUzRCdjdWRhJTNBMCcp",highlighted:`<span class="hljs-comment"># Multiple bounding boxes per image (using truck and groceries images)</span>
input_boxes = [
    [[<span class="hljs-number">75</span>, <span class="hljs-number">275</span>, <span class="hljs-number">1725</span>, <span class="hljs-number">850</span>], [<span class="hljs-number">425</span>, <span class="hljs-number">600</span>, <span class="hljs-number">700</span>, <span class="hljs-number">875</span>], [<span class="hljs-number">1375</span>, <span class="hljs-number">550</span>, <span class="hljs-number">1650</span>, <span class="hljs-number">800</span>], [<span class="hljs-number">1240</span>, <span class="hljs-number">675</span>, <span class="hljs-number">1400</span>, <span class="hljs-number">750</span>]],  <span class="hljs-comment"># Truck image: 4 boxes</span>
    [[<span class="hljs-number">450</span>, <span class="hljs-number">170</span>, <span class="hljs-number">520</span>, <span class="hljs-number">350</span>], [<span class="hljs-number">350</span>, <span class="hljs-number">190</span>, <span class="hljs-number">450</span>, <span class="hljs-number">350</span>], [<span class="hljs-number">500</span>, <span class="hljs-number">170</span>, <span class="hljs-number">580</span>, <span class="hljs-number">350</span>], [<span class="hljs-number">580</span>, <span class="hljs-number">170</span>, <span class="hljs-number">640</span>, <span class="hljs-number">350</span>]]  <span class="hljs-comment"># Groceries image: 4 boxes</span>
]
<span class="hljs-comment"># Update images for this example</span>
raw_images = [raw_images[<span class="hljs-number">0</span>], groceries_image]  <span class="hljs-comment"># truck and groceries</span>
inputs = processor(images=raw_images, input_boxes=input_boxes, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs, multimask_output=<span class="hljs-literal">False</span>)
all_masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>])
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Processed <span class="hljs-subst">{<span class="hljs-built_in">len</span>(input_boxes)}</span> images with <span class="hljs-subst">{<span class="hljs-built_in">len</span>(input_boxes[<span class="hljs-number">0</span>])}</span> and <span class="hljs-subst">{<span class="hljs-built_in">len</span>(input_boxes[<span class="hljs-number">1</span>])}</span> boxes respectively&quot;</span>)
Processed <span class="hljs-number">2</span> images <span class="hljs-keyword">with</span> <span class="hljs-number">4</span> <span class="hljs-keyword">and</span> <span class="hljs-number">4</span> boxes respectively
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;IoU scores: <span class="hljs-subst">{outputs.iou_scores.squeeze()}</span>&quot;</span>)
IoU scores: tensor([<span class="hljs-number">0.9301</span>, <span class="hljs-number">0.9348</span>, <span class="hljs-number">0.6605</span>, <span class="hljs-number">0.9465</span>], device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)`,lang:"python",wrap:!1});var q=e(D,2);s(q,{title:"Using Previous Masks as Input",local:"using-previous-masks-as-input",headingTag:"h3"});var P=e(q,4);n(P,{code:"JTIzJTIwR2V0JTIwaW5pdGlhbCUyMHNlZ21lbnRhdGlvbiUwQWlucHV0X3BvaW50cyUyMCUzRCUyMCU1QiU1QiU1QiU1QjUwMCUyQyUyMDM3NSU1RCU1RCU1RCU1RCUwQWlucHV0X2xhYmVscyUyMCUzRCUyMCU1QiU1QiU1QjElNUQlNUQlNUQlMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoaW1hZ2VzJTNEcmF3X2ltYWdlJTJDJTIwaW5wdXRfcG9pbnRzJTNEaW5wdXRfcG9pbnRzJTJDJTIwaW5wdXRfbGFiZWxzJTNEaW5wdXRfbGFiZWxzJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMpJTBBJTBBJTIzJTIwVXNlJTIwdGhlJTIwYmVzdCUyMG1hc2slMjBhcyUyMGlucHV0JTIwZm9yJTIwcmVmaW5lbWVudCUwQW1hc2tfaW5wdXQlMjAlM0QlMjBvdXRwdXRzLnByZWRfbWFza3MlNUIlM0ElMkMlMjAlM0ElMkMlMjB0b3JjaC5hcmdtYXgob3V0cHV0cy5pb3Vfc2NvcmVzLnNxdWVlemUoKSklNUQlMEElMEElMjMlMjBBZGQlMjBhZGRpdGlvbmFsJTIwcG9pbnRzJTIwd2l0aCUyMHRoZSUyMG1hc2slMjBpbnB1dCUwQW5ld19pbnB1dF9wb2ludHMlMjAlM0QlMjAlNUIlNUIlNUIlNUI1MDAlMkMlMjAzNzUlNUQlMkMlMjAlNUI0NTAlMkMlMjAzMDAlNUQlNUQlNUQlNUQlMEFuZXdfaW5wdXRfbGFiZWxzJTIwJTNEJTIwJTVCJTVCJTVCMSUyQyUyMDElNUQlNUQlNUQlMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoJTBBJTIwJTIwJTIwJTIwaW5wdXRfcG9pbnRzJTNEbmV3X2lucHV0X3BvaW50cyUyQyUwQSUyMCUyMCUyMCUyMGlucHV0X2xhYmVscyUzRG5ld19pbnB1dF9sYWJlbHMlMkMlMEElMjAlMjAlMjAlMjBvcmlnaW5hbF9zaXplcyUzRGlucHV0cyU1QiUyMm9yaWdpbmFsX3NpemVzJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUyQyUwQSkudG8obW9kZWwuZGV2aWNlKSUwQSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjByZWZpbmVkX291dHB1dHMlMjAlM0QlMjBtb2RlbCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAqKmlucHV0cyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0X21hc2tzJTNEbWFza19pbnB1dCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGltYWdlX2VtYmVkZGluZ3MlM0RvdXRwdXRzLmltYWdlX2VtYmVkZGluZ3MlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtdWx0aW1hc2tfb3V0cHV0JTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjAp",highlighted:`<span class="hljs-comment"># Get initial segmentation</span>
input_points = [[[[<span class="hljs-number">500</span>, <span class="hljs-number">375</span>]]]]
input_labels = [[[<span class="hljs-number">1</span>]]]
inputs = processor(images=raw_image, input_points=input_points, input_labels=input_labels, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(**inputs)
<span class="hljs-comment"># Use the best mask as input for refinement</span>
mask_input = outputs.pred_masks[:, :, torch.argmax(outputs.iou_scores.squeeze())]
<span class="hljs-comment"># Add additional points with the mask input</span>
new_input_points = [[[[<span class="hljs-number">500</span>, <span class="hljs-number">375</span>], [<span class="hljs-number">450</span>, <span class="hljs-number">300</span>]]]]
new_input_labels = [[[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]]
inputs = processor(
    input_points=new_input_points,
    input_labels=new_input_labels,
    original_sizes=inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>],
    return_tensors=<span class="hljs-string">&quot;pt&quot;</span>,
).to(model.device)
<span class="hljs-keyword">with</span> torch.no_grad():
    refined_outputs = model(
        **inputs,
        input_masks=mask_input,
        image_embeddings=outputs.image_embeddings,
        multimask_output=<span class="hljs-literal">False</span>,
    )`,lang:"python",wrap:!1});var K=e(P,2);s(K,{title:"EdgeTamConfig",local:"transformers.EdgeTamConfig",headingTag:"h2"});var u=e(K,2),L=l(u);t(L,{name:"class transformers.EdgeTamConfig",anchor:"transformers.EdgeTamConfig",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/configuration_edgetam.py#L152",parameters:[{name:"transformers_version",val:": str | None = None"},{name:"architectures",val:": list[str] | None = None"},{name:"output_hidden_states",val:": bool | None = False"},{name:"return_dict",val:": bool | None = True"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"chunk_size_feed_forward",val:": int = 0"},{name:"is_encoder_decoder",val:": bool = False"},{name:"id2label",val:": dict[int, str] | dict[str, str] | None = None"},{name:"label2id",val:": dict[str, int] | dict[str, str] | None = None"},{name:"problem_type",val:": Literal['regression', 'single_label_classification', 'multi_label_classification'] | None = None"},{name:"vision_config",val:": dict | transformers.configuration_utils.PreTrainedConfig | None = None"},{name:"prompt_encoder_config",val:": dict | transformers.configuration_utils.PreTrainedConfig | None = None"},{name:"mask_decoder_config",val:": dict | transformers.configuration_utils.PreTrainedConfig | None = None"},{name:"initializer_range",val:": float = 0.02"}],parametersDescription:[{anchor:"transformers.EdgeTamConfig.vision_config",description:`<strong>vision_config</strong> (<code>Union[dict, ~configuration_utils.PreTrainedConfig]</code>, <em>optional</em>) &#x2014;
The config object or dictionary of the vision backbone.`,name:"vision_config"},{anchor:"transformers.EdgeTamConfig.prompt_encoder_config",description:`<strong>prompt_encoder_config</strong> (Union[<code>dict</code>, <code>EdgeTamPromptEncoderConfig</code>], <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize <a href="/docs/transformers/pr_36708/en/model_doc/edgetam#transformers.EdgeTamPromptEncoderConfig">EdgeTamPromptEncoderConfig</a>.`,name:"prompt_encoder_config"},{anchor:"transformers.EdgeTamConfig.mask_decoder_config",description:`<strong>mask_decoder_config</strong> (Union[<code>dict</code>, <code>EdgeTamMaskDecoderConfig</code>], <em>optional</em>) &#x2014;
Dictionary of configuration options used to initialize <a href="/docs/transformers/pr_36708/en/model_doc/edgetam#transformers.EdgeTamMaskDecoderConfig">EdgeTamMaskDecoderConfig</a>.`,name:"mask_decoder_config"},{anchor:"transformers.EdgeTamConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.02</code>) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"}]});var me=e(L,6);j(me,{anchor:"transformers.EdgeTamConfig.example",children:(a,r)=>{var o=J(),d=e(p(o),2);n(d,{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjAlMjBFZGdlVGFtVmlzaW9uQ29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwRWRnZVRhbVByb21wdEVuY29kZXJDb25maWclMkMlMEElMjAlMjAlMjAlMjBFZGdlVGFtTWFza0RlY29kZXJDb25maWclMkMlMEElMjAlMjAlMjAlMjBFZGdlVGFtTW9kZWwlMkMlMEEpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEVkZ2VUYW1Db25maWclMjB3aXRoJTIwJTYwJTIyZmFjZWJvb2slMkZlZGdldGFtLjFfaGllcmFfdGlueSUyMiU2MCUyMHN0eWxlJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBFZGdlVGFtQ29uZmlnKCklMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwRWRnZVRhbU1vZGVsJTIwKHdpdGglMjByYW5kb20lMjB3ZWlnaHRzKSUyMGZyb20lMjB0aGUlMjAlNjAlMjJmYWNlYm9vayUyRmVkZ2V0YW0uMV9oaWVyYV90aW55JTIyJTYwJTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBbW9kZWwlMjAlM0QlMjBFZGdlVGFtTW9kZWwoY29uZmlndXJhdGlvbiklMEElMEElMjMlMjBBY2Nlc3NpbmclMjB0aGUlMjBtb2RlbCUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwbW9kZWwuY29uZmlnJTBBJTBBJTIzJTIwV2UlMjBjYW4lMjBhbHNvJTIwaW5pdGlhbGl6ZSUyMGElMjBFZGdlVGFtQ29uZmlnJTIwZnJvbSUyMGElMjBFZGdlVGFtVmlzaW9uQ29uZmlnJTJDJTIwRWRnZVRhbVByb21wdEVuY29kZXJDb25maWclMkMlMjBhbmQlMjBFZGdlVGFtTWFza0RlY29kZXJDb25maWclMEElMjMlMjBJbml0aWFsaXppbmclMjBFREdFVEFNJTIwdmlzaW9uJTIwZW5jb2RlciUyQyUyMG1lbW9yeSUyMGF0dGVudGlvbiUyQyUyMGFuZCUyMG1lbW9yeSUyMGVuY29kZXIlMjBjb25maWd1cmF0aW9ucyUwQXZpc2lvbl9jb25maWclMjAlM0QlMjBFZGdlVGFtVmlzaW9uQ29uZmlnKCklMEFwcm9tcHRfZW5jb2Rlcl9jb25maWclMjAlM0QlMjBFZGdlVGFtUHJvbXB0RW5jb2RlckNvbmZpZygpJTBBbWFza19kZWNvZGVyX2NvbmZpZyUyMCUzRCUyMEVkZ2VUYW1NYXNrRGVjb2RlckNvbmZpZygpJTBBJTBBY29uZmlnJTIwJTNEJTIwRWRnZVRhbUNvbmZpZyh2aXNpb25fY29uZmlnJTJDJTIwcHJvbXB0X2VuY29kZXJfY29uZmlnJTJDJTIwbWFza19kZWNvZGVyX2NvbmZpZyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
<span class="hljs-meta">... </span>    EdgeTamVisionConfig,
<span class="hljs-meta">... </span>    EdgeTamPromptEncoderConfig,
<span class="hljs-meta">... </span>    EdgeTamMaskDecoderConfig,
<span class="hljs-meta">... </span>    EdgeTamModel,
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a EdgeTamConfig with \`&quot;facebook/edgetam.1_hiera_tiny&quot;\` style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = EdgeTamConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a EdgeTamModel (with random weights) from the \`&quot;facebook/edgetam.1_hiera_tiny&quot;\` style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = EdgeTamModel(configuration)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># We can also initialize a EdgeTamConfig from a EdgeTamVisionConfig, EdgeTamPromptEncoderConfig, and EdgeTamMaskDecoderConfig</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing EDGETAM vision encoder, memory attention, and memory encoder configurations</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>vision_config = EdgeTamVisionConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt_encoder_config = EdgeTamPromptEncoderConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>mask_decoder_config = EdgeTamMaskDecoderConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>config = EdgeTamConfig(vision_config, prompt_encoder_config, mask_decoder_config)`,lang:"python",wrap:!1}),c(a,o)},$$slots:{default:!0}}),i(u);var O=e(u,2);s(O,{title:"EdgeTamVisionConfig",local:"transformers.EdgeTamVisionConfig",headingTag:"h2"});var g=e(O,2),he=l(g);t(he,{name:"class transformers.EdgeTamVisionConfig",anchor:"transformers.EdgeTamVisionConfig",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/configuration_edgetam.py#L29",parameters:[{name:"transformers_version",val:": str | None = None"},{name:"architectures",val:": list[str] | None = None"},{name:"output_hidden_states",val:": bool | None = False"},{name:"return_dict",val:": bool | None = True"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"chunk_size_feed_forward",val:": int = 0"},{name:"is_encoder_decoder",val:": bool = False"},{name:"id2label",val:": dict[int, str] | dict[str, str] | None = None"},{name:"label2id",val:": dict[str, int] | dict[str, str] | None = None"},{name:"problem_type",val:": Literal['regression', 'single_label_classification', 'multi_label_classification'] | None = None"},{name:"backbone_config",val:": dict | transformers.configuration_utils.PreTrainedConfig | None = None"},{name:"backbone_channel_list",val:": list[int] | None = None"},{name:"backbone_feature_sizes",val:": list | None = None"},{name:"fpn_hidden_size",val:": int = 256"},{name:"fpn_kernel_size",val:": int = 1"},{name:"fpn_stride",val:": int = 1"},{name:"fpn_padding",val:": int = 0"},{name:"fpn_top_down_levels",val:": list[int] | None = None"},{name:"num_feature_levels",val:": int = 3"},{name:"hidden_act",val:": str = 'gelu'"},{name:"layer_norm_eps",val:": float = 1e-06"},{name:"initializer_range",val:": float = 0.02"}],parametersDescription:[{anchor:"transformers.EdgeTamVisionConfig.backbone_config",description:`<strong>backbone_config</strong> (<code>Union[dict, ~configuration_utils.PreTrainedConfig]</code>, <em>optional</em>) &#x2014;
The configuration of the backbone model.`,name:"backbone_config"},{anchor:"transformers.EdgeTamVisionConfig.backbone_channel_list",description:`<strong>backbone_channel_list</strong> (<code>List[int]</code>, <em>optional</em>, defaults to <code>[384, 192, 96, 48]</code>) &#x2014;
The list of channel dimensions for the backbone.`,name:"backbone_channel_list"},{anchor:"transformers.EdgeTamVisionConfig.backbone_feature_sizes",description:`<strong>backbone_feature_sizes</strong> (<code>List[List[int]]</code>, <em>optional</em>, defaults to <code>[[256, 256], [128, 128], [64, 64]]</code>) &#x2014;
The spatial sizes of the feature maps from the backbone.`,name:"backbone_feature_sizes"},{anchor:"transformers.EdgeTamVisionConfig.fpn_hidden_size",description:`<strong>fpn_hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 256) &#x2014;
The hidden dimension of the FPN.`,name:"fpn_hidden_size"},{anchor:"transformers.EdgeTamVisionConfig.fpn_kernel_size",description:`<strong>fpn_kernel_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The kernel size for the convolutions in the neck.`,name:"fpn_kernel_size"},{anchor:"transformers.EdgeTamVisionConfig.fpn_stride",description:`<strong>fpn_stride</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The stride for the convolutions in the neck.`,name:"fpn_stride"},{anchor:"transformers.EdgeTamVisionConfig.fpn_padding",description:`<strong>fpn_padding</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014;
The padding for the convolutions in the neck.`,name:"fpn_padding"},{anchor:"transformers.EdgeTamVisionConfig.fpn_top_down_levels",description:`<strong>fpn_top_down_levels</strong> (<code>List[int]</code>, <em>optional</em>, defaults to <code>[2, 3]</code>) &#x2014;
The levels for the top-down FPN connections.`,name:"fpn_top_down_levels"},{anchor:"transformers.EdgeTamVisionConfig.num_feature_levels",description:`<strong>num_feature_levels</strong> (<code>int</code>, <em>optional</em>, defaults to 3) &#x2014;
The number of feature levels from the FPN to use.`,name:"num_feature_levels"},{anchor:"transformers.EdgeTamVisionConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>gelu</code>) &#x2014;
The non-linear activation function (function or string) in the decoder. For example, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;silu&quot;</code>, etc.`,name:"hidden_act"},{anchor:"transformers.EdgeTamVisionConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-06</code>) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.EdgeTamVisionConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to <code>0.02</code>) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"}]}),m(4),i(g);var $=e(g,2);s($,{title:"EdgeTamMaskDecoderConfig",local:"transformers.EdgeTamMaskDecoderConfig",headingTag:"h2"});var M=e($,2),ue=l(M);t(ue,{name:"class transformers.EdgeTamMaskDecoderConfig",anchor:"transformers.EdgeTamMaskDecoderConfig",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/configuration_edgetam.py#L113",parameters:[{name:"transformers_version",val:": str | None = None"},{name:"architectures",val:": list[str] | None = None"},{name:"output_hidden_states",val:": bool | None = False"},{name:"return_dict",val:": bool | None = True"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"chunk_size_feed_forward",val:": int = 0"},{name:"is_encoder_decoder",val:": bool = False"},{name:"id2label",val:": dict[int, str] | dict[str, str] | None = None"},{name:"label2id",val:": dict[str, int] | dict[str, str] | None = None"},{name:"problem_type",val:": Literal['regression', 'single_label_classification', 'multi_label_classification'] | None = None"},{name:"hidden_size",val:": int = 256"},{name:"hidden_act",val:": str = 'gelu'"},{name:"mlp_dim",val:": int = 2048"},{name:"num_hidden_layers",val:": int = 2"},{name:"num_attention_heads",val:": int = 8"},{name:"attention_downsample_rate",val:": int = 2"},{name:"num_multimask_outputs",val:": int = 3"},{name:"iou_head_depth",val:": int = 3"},{name:"iou_head_hidden_dim",val:": int = 256"},{name:"dynamic_multimask_via_stability",val:": bool = True"},{name:"dynamic_multimask_stability_delta",val:": float = 0.05"},{name:"dynamic_multimask_stability_thresh",val:": float = 0.98"}],parametersDescription:[{anchor:"transformers.EdgeTamMaskDecoderConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>256</code>) &#x2014;
Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.EdgeTamMaskDecoderConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>gelu</code>) &#x2014;
The non-linear activation function (function or string) in the decoder. For example, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;silu&quot;</code>, etc.`,name:"hidden_act"},{anchor:"transformers.EdgeTamMaskDecoderConfig.mlp_dim",description:`<strong>mlp_dim</strong> (<code>int</code>, <em>optional</em>, defaults to 2048) &#x2014;
The dimension of the MLP in the two-way transformer.`,name:"mlp_dim"},{anchor:"transformers.EdgeTamMaskDecoderConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to <code>2</code>) &#x2014;
Number of hidden layers in the Transformer decoder.`,name:"num_hidden_layers"},{anchor:"transformers.EdgeTamMaskDecoderConfig.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>, defaults to <code>8</code>) &#x2014;
Number of attention heads for each attention layer in the Transformer decoder.`,name:"num_attention_heads"},{anchor:"transformers.EdgeTamMaskDecoderConfig.attention_downsample_rate",description:`<strong>attention_downsample_rate</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The downsample rate for the attention layers.`,name:"attention_downsample_rate"},{anchor:"transformers.EdgeTamMaskDecoderConfig.num_multimask_outputs",description:`<strong>num_multimask_outputs</strong> (<code>int</code>, <em>optional</em>, defaults to 3) &#x2014;
The number of multimask outputs.`,name:"num_multimask_outputs"},{anchor:"transformers.EdgeTamMaskDecoderConfig.iou_head_depth",description:`<strong>iou_head_depth</strong> (<code>int</code>, <em>optional</em>, defaults to 3) &#x2014;
The depth of the IoU head.`,name:"iou_head_depth"},{anchor:"transformers.EdgeTamMaskDecoderConfig.iou_head_hidden_dim",description:`<strong>iou_head_hidden_dim</strong> (<code>int</code>, <em>optional</em>, defaults to 256) &#x2014;
The hidden dimension of the IoU head.`,name:"iou_head_hidden_dim"},{anchor:"transformers.EdgeTamMaskDecoderConfig.dynamic_multimask_via_stability",description:`<strong>dynamic_multimask_via_stability</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to use dynamic multimask via stability.`,name:"dynamic_multimask_via_stability"},{anchor:"transformers.EdgeTamMaskDecoderConfig.dynamic_multimask_stability_delta",description:`<strong>dynamic_multimask_stability_delta</strong> (<code>float</code>, <em>optional</em>, defaults to 0.05) &#x2014;
The stability delta for the dynamic multimask.`,name:"dynamic_multimask_stability_delta"},{anchor:"transformers.EdgeTamMaskDecoderConfig.dynamic_multimask_stability_thresh",description:`<strong>dynamic_multimask_stability_thresh</strong> (<code>float</code>, <em>optional</em>, defaults to 0.98) &#x2014;
The stability threshold for the dynamic multimask.`,name:"dynamic_multimask_stability_thresh"}]}),m(4),i(M);var ee=e(M,2);s(ee,{title:"EdgeTamPromptEncoderConfig",local:"transformers.EdgeTamPromptEncoderConfig",headingTag:"h2"});var b=e(ee,2),ge=l(b);t(ge,{name:"class transformers.EdgeTamPromptEncoderConfig",anchor:"transformers.EdgeTamPromptEncoderConfig",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/configuration_edgetam.py#L88",parameters:[{name:"transformers_version",val:": str | None = None"},{name:"architectures",val:": list[str] | None = None"},{name:"output_hidden_states",val:": bool | None = False"},{name:"return_dict",val:": bool | None = True"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"chunk_size_feed_forward",val:": int = 0"},{name:"is_encoder_decoder",val:": bool = False"},{name:"id2label",val:": dict[int, str] | dict[str, str] | None = None"},{name:"label2id",val:": dict[str, int] | dict[str, str] | None = None"},{name:"problem_type",val:": Literal['regression', 'single_label_classification', 'multi_label_classification'] | None = None"},{name:"hidden_size",val:": int = 256"},{name:"image_size",val:": int | list[int] | tuple[int, int] = 1024"},{name:"patch_size",val:": int | list[int] | tuple[int, int] = 16"},{name:"mask_input_channels",val:": int = 16"},{name:"num_point_embeddings",val:": int = 4"},{name:"hidden_act",val:": str = 'gelu'"},{name:"layer_norm_eps",val:": float = 1e-06"},{name:"scale",val:": int = 1"}],parametersDescription:[{anchor:"transformers.EdgeTamPromptEncoderConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to <code>256</code>) &#x2014;
Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.EdgeTamPromptEncoderConfig.image_size",description:`<strong>image_size</strong> (<code>Union[int, list[int], tuple[int, int]]</code>, <em>optional</em>, defaults to <code>1024</code>) &#x2014;
The size (resolution) of each image.`,name:"image_size"},{anchor:"transformers.EdgeTamPromptEncoderConfig.patch_size",description:`<strong>patch_size</strong> (<code>Union[int, list[int], tuple[int, int]]</code>, <em>optional</em>, defaults to <code>16</code>) &#x2014;
The size (resolution) of each patch.`,name:"patch_size"},{anchor:"transformers.EdgeTamPromptEncoderConfig.mask_input_channels",description:`<strong>mask_input_channels</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
The number of channels to be fed to the <code>MaskDecoder</code> module.`,name:"mask_input_channels"},{anchor:"transformers.EdgeTamPromptEncoderConfig.num_point_embeddings",description:`<strong>num_point_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 4) &#x2014;
The number of point embeddings to be used.`,name:"num_point_embeddings"},{anchor:"transformers.EdgeTamPromptEncoderConfig.hidden_act",description:`<strong>hidden_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>gelu</code>) &#x2014;
The non-linear activation function (function or string) in the decoder. For example, <code>&quot;gelu&quot;</code>,
<code>&quot;relu&quot;</code>, <code>&quot;silu&quot;</code>, etc.`,name:"hidden_act"},{anchor:"transformers.EdgeTamPromptEncoderConfig.layer_norm_eps",description:`<strong>layer_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1e-06</code>) &#x2014;
The epsilon used by the layer normalization layers.`,name:"layer_norm_eps"},{anchor:"transformers.EdgeTamPromptEncoderConfig.scale",description:`<strong>scale</strong> (<code>float</code>, <em>optional</em>, defaults to 1) &#x2014;
The scale factor for the prompt encoder.`,name:"scale"}]}),m(4),i(b);var se=e(b,2);s(se,{title:"EdgeTamVisionModel",local:"transformers.EdgeTamVisionModel",headingTag:"h2"});var T=e(se,2),ne=l(T);t(ne,{name:"class transformers.EdgeTamVisionModel",anchor:"transformers.EdgeTamVisionModel",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/modeling_edgetam.py#L452",parameters:[{name:"config",val:": EdgeTamVisionConfig"}],parametersDescription:[{anchor:"transformers.EdgeTamVisionModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_36708/en/model_doc/edgetam#transformers.EdgeTamVisionConfig">EdgeTamVisionConfig</a>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/pr_36708/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var ae=e(ne,8),Me=l(ae);t(Me,{name:"forward",anchor:"transformers.EdgeTamVisionModel.forward",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/modeling_edgetam.py#L470",parameters:[{name:"pixel_values",val:": typing.Optional[torch.FloatTensor] = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}]}),i(ae),i(T);var oe=e(T,2);s(oe,{title:"EdgeTamModel",local:"transformers.EdgeTamModel",headingTag:"h2"});var y=e(oe,2),te=l(y);t(te,{name:"class transformers.EdgeTamModel",anchor:"transformers.EdgeTamModel",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/modeling_edgetam.py#L945",parameters:[{name:"config",val:": EdgeTamConfig"}],parametersDescription:[{anchor:"transformers.EdgeTamModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_36708/en/model_doc/edgetam#transformers.EdgeTamConfig">EdgeTamConfig</a>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/pr_36708/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]});var f=e(te,8),le=l(f);t(le,{name:"forward",anchor:"transformers.EdgeTamModel.forward",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/modeling_edgetam.py#L1051",parameters:[{name:"pixel_values",val:": typing.Optional[torch.FloatTensor] = None"},{name:"input_points",val:": typing.Optional[torch.FloatTensor] = None"},{name:"input_labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"input_boxes",val:": typing.Optional[torch.FloatTensor] = None"},{name:"input_masks",val:": typing.Optional[torch.LongTensor] = None"},{name:"image_embeddings",val:": typing.Optional[torch.FloatTensor] = None"},{name:"multimask_output",val:": bool = True"},{name:"attention_similarity",val:": typing.Optional[torch.FloatTensor] = None"},{name:"target_embedding",val:": typing.Optional[torch.FloatTensor] = None"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.EdgeTamModel.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, image_size, image_size)</code>, <em>optional</em>) &#x2014;
The tensors corresponding to the input images. Pixel values can be obtained using
<a href="/docs/transformers/pr_36708/en/model_doc/sam2#transformers.Sam2ImageProcessor">Sam2ImageProcessor</a>. See <a href="/docs/transformers/pr_36708/en/model_doc/molmo2#transformers.Molmo2ImageProcessor.__call__">Sam2ImageProcessor.<strong>call</strong>()</a> for details (<a href="/docs/transformers/pr_36708/en/model_doc/sam2#transformers.Sam2Processor">Sam2Processor</a> uses
<a href="/docs/transformers/pr_36708/en/model_doc/sam2#transformers.Sam2ImageProcessor">Sam2ImageProcessor</a> for processing images).`,name:"pixel_values"},{anchor:"transformers.EdgeTamModel.forward.input_points",description:`<strong>input_points</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_points, 2)</code>) &#x2014;
Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much
better results. The points can be obtained by passing a list of list of list to the processor that will
create corresponding <code>torch</code> tensors of dimension 4. The first dimension is the image batch size, the
second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict
per input point), the third dimension is the number of points per segmentation mask (it is possible to pass
multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal)
coordinates of the point. If a different number of points is passed either for each image, or for each
mask, the processor will create &#x201C;PAD&#x201D; points that will correspond to the (0, 0) coordinate, and the
computation of the embedding will be skipped for these points using the labels.`,name:"input_points"},{anchor:"transformers.EdgeTamModel.forward.input_labels",description:`<strong>input_labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, point_batch_size, num_points)</code>) &#x2014;
Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the
official implementation, there are 3 types of labels</p>
<ul>
<li><code>1</code>: the point is a point that contains the object of interest</li>
<li><code>0</code>: the point is a point that does not contain the object of interest</li>
<li><code>-1</code>: the point corresponds to the background</li>
</ul>
<p>We added the label:</p>
<ul>
<li><code>-10</code>: the point is a padding point, thus should be ignored by the prompt encoder</li>
</ul>
<p>The padding labels should be automatically done by the processor.`,name:"input_labels"},{anchor:"transformers.EdgeTamModel.forward.input_boxes",description:`<strong>input_boxes</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_boxes, 4)</code>) &#x2014;
Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to
much better generated masks. The boxes can be obtained by passing a list of list of list to the processor,
that will generate a <code>torch</code> tensor, with each dimension corresponding respectively to the image batch
size, the number of boxes per image and the coordinates of the top left and bottom right point of the box.
In the order (<code>x1</code>, <code>y1</code>, <code>x2</code>, <code>y2</code>):</p>
<ul>
<li><code>x1</code>: the x coordinate of the top left point of the input box</li>
<li><code>y1</code>: the y coordinate of the top left point of the input box</li>
<li><code>x2</code>: the x coordinate of the bottom right point of the input box</li>
<li><code>y2</code>: the y coordinate of the bottom right point of the input box</li>
</ul>`,name:"input_boxes"},{anchor:"transformers.EdgeTamModel.forward.input_masks",description:`<strong>input_masks</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, image_size, image_size)</code>) &#x2014;
SAM model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to
generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be
manually fed by the user, and they need to be of shape (<code>batch_size</code>, <code>image_size</code>, <code>image_size</code>).`,name:"input_masks"},{anchor:"transformers.EdgeTamModel.forward.image_embeddings",description:`<strong>image_embeddings</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, output_channels, window_size, window_size)</code>) &#x2014;
Image embeddings, this is used by the mask decoder to generate masks and iou scores. For more memory
efficient computation, users can first retrieve the image embeddings using the <code>get_image_embeddings</code>
method, and then feed them to the <code>forward</code> method instead of feeding the <code>pixel_values</code>.`,name:"image_embeddings"},{anchor:"transformers.EdgeTamModel.forward.multimask_output",description:`<strong>multimask_output</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
In the original implementation and paper, the model always outputs 3 masks per image (or per point / per
bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the
&#x201C;best&#x201D; mask, by specifying <code>multimask_output=False</code>.`,name:"multimask_output"},{anchor:"transformers.EdgeTamModel.forward.attention_similarity",description:`<strong>attention_similarity</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the
model is used for personalization as introduced in <a href="https://huggingface.co/papers/2305.03048" rel="nofollow">PerSAM</a>.`,name:"attention_similarity"},{anchor:"transformers.EdgeTamModel.forward.target_embedding",description:`<strong>target_embedding</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case
the model is used for personalization as introduced in <a href="https://huggingface.co/papers/2305.03048" rel="nofollow">PerSAM</a>.`,name:"target_embedding"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>
<p>A <code>EdgeTamImageSegmentationOutput</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_36708/en/model_doc/edgetam#transformers.EdgeTamConfig"
>EdgeTamConfig</a>) and inputs.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>
<p><code>EdgeTamImageSegmentationOutput</code> or <code>tuple(torch.FloatTensor)</code></p>
`});var ie=e(le,4);ke(ie,{children:(a,r)=>{var o=Ee();c(a,o)},$$slots:{default:!0}});var be=e(ie,4);j(be,{anchor:"transformers.EdgeTamModel.forward.example",children:(a,r)=>{var o=J(),d=e(p(o),2);n(d,{code:"ZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1Yi51dGlscyUyMGltcG9ydCUyMGh0dHB4JTBBZnJvbSUyMGlvJTIwaW1wb3J0JTIwQnl0ZXNJTyUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWwlMkMlMjBBdXRvUHJvY2Vzc29yJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmRhbmVsY3NiJTJGZWRnZXRhbS4xX2hpZXJhX3RpbnklMjIpJTBBcHJvY2Vzc29yJTIwJTNEJTIwQXV0b1Byb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyZGFuZWxjc2IlMkZlZGdldGFtLjFfaGllcmFfdGlueSUyMiklMEElMEF1cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZodWdnaW5nZmFjZSUyRmRvY3VtZW50YXRpb24taW1hZ2VzJTJGcmVzb2x2ZSUyRm1haW4lMkZ0cmFuc2Zvcm1lcnMlMkZtb2RlbF9kb2MlMkZzYW0tY2FyLnBuZyUyMiUwQXdpdGglMjBodHRweC5zdHJlYW0oJTIyR0VUJTIyJTJDJTIwdXJsKSUyMGFzJTIwcmVzcG9uc2UlM0ElMEElMjAlMjAlMjAlMjByYXdfaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKEJ5dGVzSU8ocmVzcG9uc2UucmVhZCgpKSkuY29udmVydCglMjJSR0IlMjIpJTBBaW5wdXRfcG9pbnRzJTIwJTNEJTIwJTVCJTVCJTVCNDAwJTJDJTIwNjUwJTVEJTVEJTVEJTIwJTIwJTIzJTIwMkQlMjBsb2NhdGlvbiUyMG9mJTIwYSUyMHdpbmRvdyUyMG9uJTIwdGhlJTIwY2FyJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRHJhd19pbWFnZSUyQyUyMGlucHV0X3BvaW50cyUzRGlucHV0X3BvaW50cyUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTBBJTIzJTIwR2V0JTIwc2VnbWVudGF0aW9uJTIwbWFzayUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEElMEElMjMlMjBQb3N0cHJvY2VzcyUyMG1hc2tzJTBBbWFza3MlMjAlM0QlMjBwcm9jZXNzb3IucG9zdF9wcm9jZXNzX21hc2tzKCUwQSUyMCUyMCUyMCUyMG91dHB1dHMucHJlZF9tYXNrcyUyQyUyMGlucHV0cyU1QiUyMm9yaWdpbmFsX3NpemVzJTIyJTVEJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub.utils <span class="hljs-keyword">import</span> httpx
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> io <span class="hljs-keyword">import</span> BytesIO
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoProcessor
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModel.from_pretrained(<span class="hljs-string">&quot;danelcsb/edgetam.1_hiera_tiny&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;danelcsb/edgetam.1_hiera_tiny&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/sam-car.png&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> httpx.stream(<span class="hljs-string">&quot;GET&quot;</span>, url) <span class="hljs-keyword">as</span> response:
<span class="hljs-meta">... </span>    raw_image = Image.<span class="hljs-built_in">open</span>(BytesIO(response.read())).convert(<span class="hljs-string">&quot;RGB&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_points = [[[<span class="hljs-number">400</span>, <span class="hljs-number">650</span>]]]  <span class="hljs-comment"># 2D location of a window on the car</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(images=raw_image, input_points=input_points, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Get segmentation mask</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(**inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Postprocess masks</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>masks = processor.post_process_masks(
<span class="hljs-meta">... </span>    outputs.pred_masks, inputs[<span class="hljs-string">&quot;original_sizes&quot;</span>]
<span class="hljs-meta">... </span>)`,lang:"python",wrap:!1}),c(a,o)},$$slots:{default:!0}}),i(f);var re=e(f,2),ce=l(re);t(ce,{name:"get_image_features",anchor:"transformers.EdgeTamModel.get_image_features",source:"https://github.com/huggingface/transformers/blob/vr_36708/src/transformers/models/edgetam/modeling_edgetam.py#L1229",parameters:[{name:"pixel_values",val:": FloatTensor"},{name:"**kwargs",val:": *<class 'transformers.utils.generic.TransformersKwargs'>"}],parametersDescription:[{anchor:"transformers.EdgeTamModel.get_image_features.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code>) &#x2014;
Input pixel values of shape <code>(batch_size, num_channels, height, width)</code>.`,name:"pixel_values"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>
<p>A <code>EdgeTamVisionEncoderOutput</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_36708/en/model_doc/edgetam#transformers.EdgeTamConfig"
>EdgeTamConfig</a>) and inputs.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>
<p><code>EdgeTamVisionEncoderOutput</code> or <code>tuple(torch.FloatTensor)</code></p>
`});var Te=e(ce,4);j(Te,{anchor:"transformers.EdgeTamModel.get_image_features.example",children:(a,r)=>{var o=J(),d=e(p(o),2);n(d,{code:"",highlighted:"",lang:"python",wrap:!1}),c(a,o)},$$slots:{default:!0}}),i(re),i(y);var ye=e(y,2);Ue(ye,{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/edgetam.md"}),m(2),c(de,U),Ie()}export{Re as component};
Xet Storage Details

Size:: 89.5 kB
Xet hash:: 8591e88fe0c7ca1b9a39d33f2d45139fda1b1b4e2eb818e1cc8994d92414b0e6
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.