Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / transformers /pr_33892 /en /tasks /training_vision_backbone.html

rtrm

about 1 month ago

download

raw

36.6 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Training Vision Models using Backbone API","local":"training-vision-models-using-backbone-api","sections":[],"depth":1}">
	<link href="/docs/transformers/pr_33892/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/entry/start.b2c4257a.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/scheduler.31fdf58d.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/singletons.9860629f.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/index.252883d5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/paths.e85c0ec8.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/entry/app.05ef1f97.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/preload-helper.40847a0e.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/index.2f76fdf0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/nodes/0.ca4aafa4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/nodes/575.fd2f0e67.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/CopyLLMTxtMenu.ff482081.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.71f274cc.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/IconCopy.ac192424.js">
	<link rel="modulepreload" href="/docs/transformers/pr_33892/en/_app/immutable/chunks/CodeBlock.ab12f8e1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Training Vision Models using Backbone API","local":"training-vision-models-using-backbone-api","sections":[],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 max-sm:gap-0.5 h-6 max-sm:h-5 px-2 max-sm:px-1.5 text-[11px] max-sm:text-[9px] font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0"><svg class="w-3 h-3 max-sm:w-2.5 max-sm:h-2.5" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-6 max-sm:h-5 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible w-3 h-3 max-sm:w-2.5 max-sm:h-2.5 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="training-vision-models-using-backbone-api" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#training-vision-models-using-backbone-api"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Training Vision Models using Backbone API</span></h1> <p data-svelte-h="svelte-o5q19n">Computer vision workflows follow a common pattern. Use a pre-trained backbone for feature extraction (<a href="../model_doc/vit">ViT</a>, <a href="../model_doc/dinov3">DINOv3</a>). Add a “neck” for feature enhancement. Attach a task-specific head (<a href="../model_doc/detr">DETR</a> for object detection, <a href="../model_doc/maskformer">MaskFormer</a> for segmentation).</p> <p data-svelte-h="svelte-epst0l">The Transformers library implements these models and the <a href="../backbones">backbone API</a> lets you swap different backbones and heads with minimal code.</p> <p data-svelte-h="svelte-eisybf"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/Backbone.png" alt="Backbone Explanation"></p> <p data-svelte-h="svelte-41fxtd">This guide combines <a href="https://huggingface.co/facebook/dinov3-convnext-large-pretrain-lvd1689m" rel="nofollow">DINOv3 with ConvNext architecture</a> and a <a href="https://huggingface.co/facebook/detr-resnet-50" rel="nofollow">DETR head</a>. You’ll train on the <a href="https://huggingface.co/datasets/merve/license-plates" rel="nofollow">license plate detection dataset</a>. DINOv3 delivers the best performance as of this writing.</p> <blockquote class="note" data-svelte-h="svelte-4iz4lk"><p>This model requires access approval. Visit <a href="https://huggingface.co/facebook/dinov3-convnext-large-pretrain-lvd1689m" rel="nofollow">the model repository</a> to request access.</p></blockquote> <p data-svelte-h="svelte-byikjs">Install <a href="https://github.com/gradio-app/trackio" rel="nofollow">trackio</a> for experiment tracking and <a href="https://albumentations.ai/" rel="nofollow">albumentations</a> for data augmentation. Use the latest transformers version.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install -Uq albumentations trackio transformers datasets<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1t4xzd1">Initialize <a href="/docs/transformers/pr_33892/en/model_doc/detr#transformers.DetrConfig">DetrConfig</a> with the pre-trained DINOv3 ConvNext backbone. Use <code>num_labels=1</code> to detect the license plate bounding boxes. Create <a href="/docs/transformers/pr_33892/en/model_doc/detr#transformers.DetrForObjectDetection">DetrForObjectDetection</a> with this configuration. Freeze the backbone to preserve DINOv3 features without updating weights. Load the <a href="/docs/transformers/pr_33892/en/model_doc/detr#transformers.DetrImageProcessor">DetrImageProcessor</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DetrConfig, DetrForObjectDetection, AutoImageProcessor

	config = DetrConfig(backbone=<span class="hljs-string">"facebook/dinov3-convnext-large-pretrain-lvd1689m"</span>,
	use_pretrained_backbone=<span class="hljs-literal">True</span>, use_timm_backbone=<span class="hljs-literal">False</span>,
	num_labels=<span class="hljs-number">1</span>, id2label={<span class="hljs-number">0</span>: <span class="hljs-string">"license_plate"</span>}, label2id={<span class="hljs-string">"license_plate"</span>: <span class="hljs-number">0</span>})
	model = DetrForObjectDetection(config)

	<span class="hljs-keyword">for</span> param <span class="hljs-keyword">in</span> model.model.backbone.parameters():
	param.requires_grad = <span class="hljs-literal">False</span>
	image_processor = AutoImageProcessor.from_pretrained(<span class="hljs-string">"facebook/detr-resnet-50"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-wctl5s">Load the dataset and split it for training.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
	ds = load_dataset(<span class="hljs-string">"merve/license-plates"</span>)
	ds = ds[<span class="hljs-string">"train"</span>]

	ds = ds.train_test_split(test_size=<span class="hljs-number">0.05</span>)
	train_dataset = ds[<span class="hljs-string">"train"</span>]
	val_dataset = ds[<span class="hljs-string">"test"</span>]
	<span class="hljs-built_in">len</span>(train_dataset)
	<span class="hljs-comment"># 5867</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1xdtmf0">Augment the dataset. Rescale images to a maximum size, flip them, and apply affine transforms. Eliminate invalid bounding boxes and ensure annotations stay clean with <code>rebuild_objects</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> albumentations <span class="hljs-keyword">as</span> A
	<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
	<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

	train_aug = A.Compose(
	[
	A.LongestMaxSize(max_size=<span class="hljs-number">1024</span>, p=<span class="hljs-number">1.0</span>),
	A.HorizontalFlip(p=<span class="hljs-number">0.5</span>),
	A.Affine(rotate=(-<span class="hljs-number">5</span>, <span class="hljs-number">5</span>), shear=(-<span class="hljs-number">5</span>, <span class="hljs-number">5</span>), translate_percent=(<span class="hljs-number">0.05</span>, <span class="hljs-number">0.05</span>), p=<span class="hljs-number">0.5</span>),
	],
	bbox_params=A.BboxParams(<span class="hljs-built_in">format</span>=<span class="hljs-string">"coco"</span>, label_fields=[<span class="hljs-string">"category_id"</span>], min_visibility=<span class="hljs-number">0.0</span>),
	)

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">train_transform</span>(<span class="hljs-params">batch</span>):
	imgs_out, objs_out = [], []
	original_imgs, original_objs = batch[<span class="hljs-string">"image"</span>], batch[<span class="hljs-string">"objects"</span>]

	<span class="hljs-keyword">for</span> i, (img_pil, objs) <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(<span class="hljs-built_in">zip</span>(original_imgs, original_objs)):
	img = np.array(img_pil)
	labels = [<span class="hljs-number">0</span>] * <span class="hljs-built_in">len</span>(objs[<span class="hljs-string">"bbox"</span>])

	out = train_aug(image=img, bboxes=<span class="hljs-built_in">list</span>(objs[<span class="hljs-string">"bbox"</span>]), category_id=labels)

	<span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(out[<span class="hljs-string">"bboxes"</span>]) == <span class="hljs-number">0</span>:
	imgs_out.append(img_pil) <span class="hljs-comment"># if no boxes left after augmentation, use original</span>
	objs_out.append(objs)
	<span class="hljs-keyword">continue</span>

	H, W = out[<span class="hljs-string">"image"</span>].shape[:<span class="hljs-number">2</span>]
	clamped = []
	<span class="hljs-keyword">for</span> (x, y, w, h) <span class="hljs-keyword">in</span> out[<span class="hljs-string">"bboxes"</span>]:
	x = <span class="hljs-built_in">max</span>(<span class="hljs-number">0.0</span>, <span class="hljs-built_in">min</span>(x, W - <span class="hljs-number">1.0</span>))
	y = <span class="hljs-built_in">max</span>(<span class="hljs-number">0.0</span>, <span class="hljs-built_in">min</span>(y, H - <span class="hljs-number">1.0</span>))
	w = <span class="hljs-built_in">max</span>(<span class="hljs-number">1.0</span>, <span class="hljs-built_in">min</span>(w, W - x))
	h = <span class="hljs-built_in">max</span>(<span class="hljs-number">1.0</span>, <span class="hljs-built_in">min</span>(h, H - y))
	clamped.append([x, y, w, h])

	imgs_out.append(Image.fromarray(out[<span class="hljs-string">"image"</span>]))
	objs_out.append(rebuild_objects(clamped, out[<span class="hljs-string">"category_id"</span>]))

	batch[<span class="hljs-string">"image"</span>] = imgs_out
	batch[<span class="hljs-string">"objects"</span>] = objs_out
	<span class="hljs-keyword">return</span> batch



	<span class="hljs-keyword">def</span> <span class="hljs-title function_">rebuild_objects</span>(<span class="hljs-params">bboxes, labels</span>):
	bboxes = [<span class="hljs-built_in">list</span>(<span class="hljs-built_in">map</span>(<span class="hljs-built_in">float</span>, b)) <span class="hljs-keyword">for</span> b <span class="hljs-keyword">in</span> bboxes]
	areas = [<span class="hljs-built_in">float</span>(w*h) <span class="hljs-keyword">for</span> (_, _, w, h) <span class="hljs-keyword">in</span> bboxes]
	ids = <span class="hljs-built_in">list</span>(<span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(bboxes)))
	<span class="hljs-keyword">return</span> {
	<span class="hljs-string">"id"</span>: ids,
	<span class="hljs-string">"bbox"</span>: bboxes,
	<span class="hljs-string">"category_id"</span>: <span class="hljs-built_in">list</span>(<span class="hljs-built_in">map</span>(<span class="hljs-built_in">int</span>, labels)),
	<span class="hljs-string">"area"</span>: areas,
	<span class="hljs-string">"iscrowd"</span>: [<span class="hljs-number">0</span>]*<span class="hljs-built_in">len</span>(bboxes),
	}

	train_dataset = train_dataset.with_transform(train_transform)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-53ge6v">Build COCO-style annotations for the image processor.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">format_annotations</span>(<span class="hljs-params">image, objects, image_id</span>):
	n = <span class="hljs-built_in">len</span>(objects[<span class="hljs-string">"id"</span>])
	anns = []
	iscrowd_list = objects.get(<span class="hljs-string">"iscrowd"</span>, [<span class="hljs-number">0</span>] * n)
	area_list = objects.get(<span class="hljs-string">"area"</span>, <span class="hljs-literal">None</span>)

	<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n):
	x, y, w, h = objects[<span class="hljs-string">"bbox"</span>][i]
	area = area_list[i] <span class="hljs-keyword">if</span> area_list <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">else</span> <span class="hljs-built_in">float</span>(w * h)

	anns.append({
	<span class="hljs-string">"id"</span>: <span class="hljs-built_in">int</span>(objects[<span class="hljs-string">"id"</span>][i]),
	<span class="hljs-string">"iscrowd"</span>: <span class="hljs-built_in">int</span>(iscrowd_list[i]),
	<span class="hljs-string">"bbox"</span>: [<span class="hljs-built_in">float</span>(x), <span class="hljs-built_in">float</span>(y), <span class="hljs-built_in">float</span>(w), <span class="hljs-built_in">float</span>(h)],
	<span class="hljs-string">"category_id"</span>: <span class="hljs-built_in">int</span>(objects.get(<span class="hljs-string">"category_id"</span>, objects.get(<span class="hljs-string">"category"</span>))[i]),
	<span class="hljs-string">"area"</span>: <span class="hljs-built_in">float</span>(area),
	})

	<span class="hljs-keyword">return</span> {<span class="hljs-string">"image_id"</span>: <span class="hljs-built_in">int</span>(image_id), <span class="hljs-string">"annotations"</span>: anns}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1vmjjmf">Create batches in the data collator. Format annotations and pass them with transformed images to the image processor.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">collate_fn</span>(<span class="hljs-params">examples</span>):
	images = [example[<span class="hljs-string">"image"</span>] <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> examples]
	ann_batch = [format_annotations(example[<span class="hljs-string">"image"</span>], example[<span class="hljs-string">"objects"</span>], example[<span class="hljs-string">"image_id"</span>]) <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> examples]

	inputs = image_processor(images=images, annotations=ann_batch, return_tensors=<span class="hljs-string">"pt"</span>)
	<span class="hljs-keyword">return</span> inputs<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-7bu0oa">Initialize the <a href="/docs/transformers/pr_33892/en/main_classes/trainer#transformers.Trainer">Trainer</a> and set up <a href="/docs/transformers/pr_33892/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a> for model convergence. Pass datasets, data collator, arguments, and model to <code>Trainer</code> to start training.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

	training_args = TrainingArguments(
	output_dir=<span class="hljs-string">"./license-plate-detr-dinov3"</span>,
	per_device_train_batch_size=<span class="hljs-number">4</span>,
	per_device_eval_batch_size=<span class="hljs-number">4</span>,
	num_train_epochs=<span class="hljs-number">8</span>,
	learning_rate=<span class="hljs-number">1e-5</span>,
	weight_decay=<span class="hljs-number">1e-4</span>,
	warmup_steps=<span class="hljs-number">500</span>,
	eval_strategy=<span class="hljs-string">"steps"</span>,
	eval_steps=<span class="hljs-number">500</span>,
	save_total_limit=<span class="hljs-number">2</span>,
	dataloader_pin_memory=<span class="hljs-literal">False</span>,
	fp16=<span class="hljs-literal">True</span>,
	report_to=<span class="hljs-string">"trackio"</span>,
	load_best_model_at_end=<span class="hljs-literal">True</span>,
	remove_unused_columns=<span class="hljs-literal">False</span>,
	push_to_hub=<span class="hljs-literal">True</span>,
	)

	trainer = Trainer(
	model=model,
	args=training_args,
	train_dataset=train_dataset,
	eval_dataset=val_dataset,
	data_collator=collate_fn,
	)

	trainer.train()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-2uuh0t">Push the trainer and image processor to the Hub.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->trainer.push_to_hub()
	image_processor.push_to_hub(<span class="hljs-string">"merve/license-plate-detr-dinov3"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-idgumf">Test the model with an object detection pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

	obj_detector = pipeline(
	<span class="hljs-string">"object-detection"</span>, model=<span class="hljs-string">"merve/license-plate-detr-dinov3"</span>
	)
	results = obj_detector(<span class="hljs-string">"https://huggingface.co/datasets/merve/vlm_test_images/resolve/main/license-plates.jpg"</span>, threshold=<span class="hljs-number">0.05</span>)
	<span class="hljs-built_in">print</span>(results)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-nnk1op">Visualize the results.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image, ImageDraw
	<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
	<span class="hljs-keyword">import</span> requests


	<span class="hljs-keyword">def</span> <span class="hljs-title function_">plot_results</span>(<span class="hljs-params">image, results, threshold</span>):
	image = Image.fromarray(np.uint8(image))
	draw = ImageDraw.Draw(image)
	width, height = image.size

	<span class="hljs-keyword">for</span> result <span class="hljs-keyword">in</span> results:
	score = result[<span class="hljs-string">"score"</span>]
	label = result[<span class="hljs-string">"label"</span>]
	box = <span class="hljs-built_in">list</span>(result[<span class="hljs-string">"box"</span>].values())

	<span class="hljs-keyword">if</span> score > threshold:
	x1, y1, x2, y2 = <span class="hljs-built_in">tuple</span>(box)
	draw.rectangle((x1, y1, x2, y2), outline=<span class="hljs-string">"red"</span>)
	draw.text((x1 + <span class="hljs-number">5</span>, y1 + <span class="hljs-number">10</span>), <span class="hljs-string">f"<span class="hljs-subst">{score:<span class="hljs-number">.2</span>f}</span>"</span>, fill=<span class="hljs-string">"green"</span> <span class="hljs-keyword">if</span> score > <span class="hljs-number">0.7</span> <span class="hljs-keyword">else</span> <span class="hljs-string">"red"</span>)

	<span class="hljs-keyword">return</span> image

	image = Image.<span class="hljs-built_in">open</span>(requests.get(<span class="hljs-string">"https://huggingface.co/datasets/merve/vlm_test_images/resolve/main/license-plates.jpg"</span>, stream=<span class="hljs-literal">True</span>).raw)
	plot_results(image, results, threshold=<span class="hljs-number">0.05</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-kxu6oq"><img src="https://huggingface.co/datasets/huggingface/documentation-images/results/main/transformers/tasks/backbone_training_results.png" alt="Results"></p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/tasks/training_vision_backbone.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_16tnnm8 = {
	assets: "/docs/transformers/pr_33892/en",
	base: "/docs/transformers/pr_33892/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/transformers/pr_33892/en/_app/immutable/entry/start.b2c4257a.js"),
	import("/docs/transformers/pr_33892/en/_app/immutable/entry/app.05ef1f97.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 575],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 36.6 kB
Xet hash:: c10d97f3218c35729be484339ab7863576938892c02895cb4b3e0e0c0cd97991

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.