Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / transformers /pr_43265 /en /trainer_recipes.html

HuggingFaceDocBuilder

about 1 month ago

download

raw

57.2 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Trainer features","local":"trainer-features","sections":[{"title":"Custom loss function","local":"custom-loss-function","sections":[],"depth":2},{"title":"Evaluating on start","local":"evaluating-on-start","sections":[],"depth":2},{"title":"Memory-efficient evals","local":"memory-efficient-evals","sections":[{"title":"eval_accumulation_steps","local":"evalaccumulationsteps","sections":[],"depth":3},{"title":"preprocess_logits_for_metrics","local":"preprocesslogitsformetrics","sections":[],"depth":3}],"depth":2},{"title":"Dataloader performance","local":"dataloader-performance","sections":[],"depth":2},{"title":"NEFTune","local":"neftune","sections":[],"depth":2},{"title":"Logging","local":"logging","sections":[],"depth":2},{"title":"Checkpointing","local":"checkpointing","sections":[{"title":"Resume training","local":"resume-training","sections":[],"depth":3},{"title":"JIT checkpointing","local":"jit-checkpointing","sections":[],"depth":3}],"depth":2}],"depth":1}">
	<link href="/docs/transformers/pr_43265/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/entry/start.1dae3980.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/scheduler.31fdf58d.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/singletons.5fcb8d34.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/index.252883d5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/paths.e648cef5.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/entry/app.66dba7a3.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/preload-helper.5b8be175.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/index.2f76fdf0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/nodes/0.4837e553.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/nodes/713.e66c2acb.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/CopyLLMTxtMenu.53b607bf.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.08750ec0.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/IconCopy.ac192424.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/CodeBlock.e52df5d6.js">
	<link rel="modulepreload" href="/docs/transformers/pr_43265/en/_app/immutable/chunks/HfOption.fb051768.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Trainer features","local":"trainer-features","sections":[{"title":"Custom loss function","local":"custom-loss-function","sections":[],"depth":2},{"title":"Evaluating on start","local":"evaluating-on-start","sections":[],"depth":2},{"title":"Memory-efficient evals","local":"memory-efficient-evals","sections":[{"title":"eval_accumulation_steps","local":"evalaccumulationsteps","sections":[],"depth":3},{"title":"preprocess_logits_for_metrics","local":"preprocesslogitsformetrics","sections":[],"depth":3}],"depth":2},{"title":"Dataloader performance","local":"dataloader-performance","sections":[],"depth":2},{"title":"NEFTune","local":"neftune","sections":[],"depth":2},{"title":"Logging","local":"logging","sections":[],"depth":2},{"title":"Checkpointing","local":"checkpointing","sections":[{"title":"Resume training","local":"resume-training","sections":[],"depth":3},{"title":"JIT checkpointing","local":"jit-checkpointing","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="trainer-features" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#trainer-features"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Trainer features</span></h1> <p data-svelte-h="svelte-1qbi7dv">Each recipe below demonstrates a specific <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> feature: custom loss functions, memory-efficient evaluation, checkpointing strategies, and more.</p> <blockquote class="tip" data-svelte-h="svelte-1nu0n38"><p>Open an <a href="https://github.com/huggingface/transformers/issues/new/choose" rel="nofollow">issue</a> if there is a feature or workflow you’d like to see here.</p></blockquote> <h2 class="relative group"><a id="custom-loss-function" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#custom-loss-function"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Custom loss function</span></h2> <p data-svelte-h="svelte-gm1fhx">Pass <a href="https://huggingface.co/docs/transformers/en/main_classes/trainer#transformers.Trainer.compute_loss_func" rel="nofollow">compute_loss_func</a> to <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> to replace the default loss function. The function runs <em>after</em> the forward pass and only defines how loss is computed from the outputs. To modify the forward pass itself, <a href="./trainer_customize#compute_loss">subclass</a> <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer.compute_loss">compute_loss()</a> instead.</p> <p data-svelte-h="svelte-1ipu6gi">The custom loss function must have the following signature:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch.nn.functional <span class="hljs-keyword">as</span> F

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">my_loss_fn</span>(<span class="hljs-params">outputs, labels, num_items_in_batch</span>):
	logits = outputs[<span class="hljs-string">"logits"</span>]
	loss = F.cross_entropy(logits, labels, reduction=<span class="hljs-string">"sum"</span>)
	<span class="hljs-keyword">return</span> loss / num_items_in_batch<!-- HTML_TAG_END --></pre></div> <ul data-svelte-h="svelte-1ulnw44"><li><code>outputs</code> is the raw model output (<code>outputs.logits</code> has shape <code>(batch, seq_len, vocab_size)</code>).</li> <li><code>labels</code> is the token ids popped from the input batch by <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> before the forward pass.</li> <li><code>num_items_in_batch</code> is the number of prediction targets across the full accumulated batch. For causal LM models it counts the shifted labels (<code>labels[..., 1:]</code>), since the label shift leaves position 0 of every sequence without a target. See <a href="./grad_accumulation#loss-scaling">Loss scaling</a> for details. <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> skips automatic loss normalization when a custom loss function is provided, so your function must handle normalization directly.</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START -->trainer = Trainer(
	model=model,
	args=TrainingArguments(...),
	train_dataset=train_dataset,
	compute_loss_func=my_loss_fn,
	)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <blockquote class="note" data-svelte-h="svelte-1s8dz4m"><p>See the <a href="./trainer_customize#compute_loss">subclassing guide</a> for more examples of overriding <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer.compute_loss">compute_loss()</a>.</p></blockquote> <h2 class="relative group"><a id="evaluating-on-start" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#evaluating-on-start"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Evaluating on start</span></h2> <p data-svelte-h="svelte-saeu99">Set <code>eval_on_start=True</code> to run a full eval pass before the first training step. A pre-training eval surfaces issues with the evaluation pipeline early, especially during long runs.</p> <p data-svelte-h="svelte-1yepp3e"><code>eval_on_start</code> requires a valid <code>eval_strategy</code> (such as <code>"epoch"</code>) and an eval dataset.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

	trainer = Trainer(
	model=model,
	args=TrainingArguments(
	output_dir=<span class="hljs-string">"out"</span>,
	eval_strategy=<span class="hljs-string">"epoch"</span>,
	eval_on_start=<span class="hljs-literal">True</span>,
	),
	train_dataset=train_dataset,
	eval_dataset=eval_dataset,
	compute_metrics=compute_metrics,
	)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-18ivscr">A full eval adds time, so it’s most useful on first runs or after modifying <code>compute_metrics</code>.</p> <h2 class="relative group"><a id="memory-efficient-evals" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#memory-efficient-evals"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Memory-efficient evals</span></h2> <p data-svelte-h="svelte-158uxlf">During evaluation, <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> runs a forward pass on every batch and concatenates the logits into a single tensor on the GPU. Once the eval dataset is fully processed, <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> moves the concatenated logits to the CPU and calls <code>compute_metrics</code>. For large models or eval sets, the accumulated logits can exhaust GPU memory even when training on the same hardware works fine, because training only holds one batch of activations at a time.</p> <h3 class="relative group"><a id="evalaccumulationsteps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#evalaccumulationsteps"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>eval_accumulation_steps</span></h3> <p data-svelte-h="svelte-1j6ku5i">Offload the accumulated predictions from GPU to CPU every <em>n</em> batches. Lower values reduce GPU memory at the cost of more frequent CPU transfers.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

	trainer = Trainer(
	model=model,
	args=TrainingArguments(
	output_dir=<span class="hljs-string">"out"</span>,
	eval_strategy=<span class="hljs-string">"epoch"</span>,
	eval_accumulation_steps=<span class="hljs-number">16</span>, <span class="hljs-comment"># move predictions to CPU every 16 batches</span>
	),
	eval_dataset=eval_dataset,
	compute_metrics=compute_metrics,
	)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="preprocesslogitsformetrics" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#preprocesslogitsformetrics"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>preprocess_logits_for_metrics</span></h3> <p data-svelte-h="svelte-19jtfuj">Called once per eval batch on the GPU, immediately after the forward pass and before logit accumulation. The returned value replaces the logits in <code>eval_pred.predictions</code>. Running the computation at the batch level reduces per-batch tensor size and gives <code>eval_accumulation_steps</code> a smaller tensor to offload.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> evaluate
	<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

	metric = evaluate.load(<span class="hljs-string">"accuracy"</span>)

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_logits_for_metrics</span>(<span class="hljs-params">logits, labels</span>):
	<span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(logits, <span class="hljs-built_in">tuple</span>):
	logits = logits[<span class="hljs-number">0</span>]
	<span class="hljs-keyword">return</span> logits.argmax(dim=-<span class="hljs-number">1</span>)

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_preds</span>):
	preds, labels = eval_preds
	labels = labels[:, <span class="hljs-number">1</span>:].reshape(-<span class="hljs-number">1</span>)
	preds = preds[:, :-<span class="hljs-number">1</span>].reshape(-<span class="hljs-number">1</span>)
	<span class="hljs-keyword">return</span> metric.compute(predictions=preds, references=labels)

	trainer = Trainer(
	model=model,
	args=TrainingArguments(
	output_dir=<span class="hljs-string">"out"</span>,
	eval_strategy=<span class="hljs-string">"epoch"</span>,
	eval_accumulation_steps=<span class="hljs-number">16</span>,
	),
	eval_dataset=eval_dataset,
	compute_metrics=compute_metrics,
	preprocess_logits_for_metrics=preprocess_logits_for_metrics,
	)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="dataloader-performance" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dataloader-performance"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Dataloader performance</span></h2> <p data-svelte-h="svelte-7gi6dv">By default, <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> creates a dataloader with <code>dataloader_num_workers=0</code>. Data is loaded in the main process while the GPU idles, which shows up as low GPU utilization between batches.</p> <p data-svelte-h="svelte-dt77rq">Both <code>dataloader_persistent_workers</code> and <code>dataloader_prefetch_factor</code> require <code>dataloader_num_workers > 0</code>.</p> <ul data-svelte-h="svelte-1gbbe9t"><li><code>dataloader_persistent_workers</code> keeps worker subprocesses alive between epochs to avoid reinitializing from scratch, at the cost of higher memory.</li> <li><code>dataloader_prefetch_factor</code> controls how many batches each worker prepares in advance. With <code>dataloader_prefetch_factor=2</code> and <code>num_workers=4</code>, up to 8 batches sit in memory while the GPU trains on the current one.</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

	args = TrainingArguments(
	output_dir=<span class="hljs-string">"out"</span>,
	dataloader_num_workers=<span class="hljs-number">4</span>, <span class="hljs-comment"># spawn 4 worker subprocesses</span>
	dataloader_persistent_workers=<span class="hljs-literal">True</span>, <span class="hljs-comment"># keep them alive between epochs</span>
	dataloader_prefetch_factor=<span class="hljs-number">2</span>, <span class="hljs-comment"># each worker preloads 2 batches ahead</span>
	)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="neftune" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#neftune"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>NEFTune</span></h2> <p data-svelte-h="svelte-3g1y0q"><a href="https://hf.co/papers/2310.05914" rel="nofollow">NEFTune</a> adds random noise to token embeddings during the forward pass. The noise acts as regularization and can improve performance for instruction fine-tuning.</p> <p data-svelte-h="svelte-aho6ln">Enable NEFTune by setting <code>neftune_noise_alpha</code> in <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>. Typical alpha values range from 5 to 15.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

	trainer = Trainer(
	model=model,
	args=TrainingArguments(
	output_dir=<span class="hljs-string">"out"</span>,
	num_train_epochs=<span class="hljs-number">3</span>,
	neftune_noise_alpha=<span class="hljs-number">5</span>,
	),
	train_dataset=train_dataset,
	)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1knfcd4">NEFTune only affects training, and the original embedding layer is restored after training.</p> <h2 class="relative group"><a id="logging" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#logging"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Logging</span></h2> <p data-svelte-h="svelte-1ggid81">Control when and where <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> writes log entries with <code>logging_strategy</code>, <code>logging_steps</code>, and <code>report_to</code>.</p> <ul data-svelte-h="svelte-1we43ai"><li><code>logging_strategy="steps"</code> logs every <code>logging_steps()</code> optimizer updates. Use <code>"epoch"</code> to log at each epoch end instead.</li> <li><code>report_to</code> streams logs to an experiment tracker like Trackio.</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

	trainer = Trainer(
	model=model,
	args=TrainingArguments(
	output_dir=<span class="hljs-string">"out"</span>,
	logging_strategy=<span class="hljs-string">"steps"</span>,
	logging_steps=<span class="hljs-number">50</span>, <span class="hljs-comment"># write a log entry every 50 optimizer updates</span>
	report_to=<span class="hljs-string">"trackio"</span>, <span class="hljs-comment"># stream to Trackio (or "wandb", "tensorboard", …)</span>
	run_name=<span class="hljs-string">"model-experiment-v1"</span>, <span class="hljs-comment"># display name in the tracker</span>
	),
	train_dataset=train_dataset,
	)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="checkpointing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#checkpointing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Checkpointing</span></h2> <p data-svelte-h="svelte-3l0r1o"><a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> saves a checkpoint every <code>save_steps()</code> optimizer update and keeps all of them (or the most recent <code>~TrainingArguments.save_total_limit</code>).</p> <p data-svelte-h="svelte-1ebp3ke"><code>save_strategy="best"</code> keeps only the single best checkpoint according to a metric. A new checkpoint is saved only when the tracked metric improves, which saves disk space and avoids accumulating stale checkpoints.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer, TrainingArguments

	trainer = Trainer(
	model=model,
	args=TrainingArguments(
	output_dir=<span class="hljs-string">"out"</span>,
	eval_strategy=<span class="hljs-string">"epoch"</span>,
	save_strategy=<span class="hljs-string">"best"</span>,
	metric_for_best_model=<span class="hljs-string">"perplexity"</span>, <span class="hljs-comment"># save when eval perplexity improves</span>
	greater_is_better=<span class="hljs-literal">False</span>, <span class="hljs-comment"># lower perplexity is better</span>
	load_best_model_at_end=<span class="hljs-literal">True</span>, <span class="hljs-comment"># load the best weights after training finishes</span>
	),
	train_dataset=train_dataset,
	eval_dataset=eval_dataset,
	compute_metrics=compute_metrics, <span class="hljs-comment"># must return {"perplexity": ...}</span>
	)
	trainer.train()<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="resume-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resume-training"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Resume training</span></h3> <p data-svelte-h="svelte-1ao86j2">Pass <code>resume_from_checkpoint=True</code> to <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer.train">train()</a> if training was interrupted and you’d like to resume without losing progress. Training will resume from the latest checkpoint in <code>output_dir</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START -->trainer.train(resume_from_checkpoint=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-g721ov">Specify a checkpoint path to resume from a particular point.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START -->trainer.train(resume_from_checkpoint=<span class="hljs-string">"out/checkpoint-1000"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1mhoyeh">When resuming, <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> restores the optimizer state, scheduler state, and RNG state.</p> <p data-svelte-h="svelte-wyi1rr">Checkpoint resuming requires optimizer and scheduler state files in the checkpoint directory. If those files are missing (for example, when <code>save_only_model=True</code>), the optimizer restarts from scratch.</p> <h3 class="relative group"><a id="jit-checkpointing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#jit-checkpointing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>JIT checkpointing</span></h3> <p data-svelte-h="svelte-ela3wq">With periodic checkpointing (save_strategy=“steps” or “epoch”), you lose any training progress between the last saved checkpoint and an interruption. On shared clusters with preemptible workloads such as <a href="https://kueue.sigs.k8s.io/" rel="nofollow">Kueue</a>, jobs can be terminated at any time, so that gap can mean hours of wasted compute.</p> <p data-svelte-h="svelte-1f2med">JIT (Just-In-Time) checkpointing closes this gap. When the trainer receives a SIGTERM signal, it saves a checkpoint at the exact point training was interrupted, so you resume with minimal loss of progress. It works alongside periodic checkpointing. Periodic saves guard against crashes and hardware failures, while JIT saves guard against preemption and graceful shutdowns.</p> <p data-svelte-h="svelte-178rjwo">Enable it by setting <code>enable_jit_checkpoint=True</code> in <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

	training_args = TrainingArguments(
	output_dir=<span class="hljs-string">"your-model"</span>,
	enable_jit_checkpoint=<span class="hljs-literal">True</span>,
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-r7tuj9">When SIGTERM is received, <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> waits for the current training step to finish, saves a checkpoint, and stops training gracefully. A sentinel file (<code>checkpoint-is-incomplete.txt</code>) is written when the save begins and removed once the checkpoint is fully written. If a checkpoint directory still contains this file, the save was interrupted before completing. <a href="/docs/transformers/pr_43265/en/main_classes/trainer#transformers.Trainer">Trainer</a> doesn’t check for it automatically, so inspect for it yourself before resuming.</p> <p data-svelte-h="svelte-1vq81b7">Resume from the JIT checkpoint the same way as any other checkpoint.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-py "><!-- HTML_TAG_START -->trainer.train(resume_from_checkpoint=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <blockquote class="warning" data-svelte-h="svelte-675qgr"><p>You must configure your orchestrator to allow enough time for the checkpoint to complete. The default Kubernetes graceful shutdown period is only 30 seconds, which is typically not enough for larger models.</p></blockquote> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Kubernetes </div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Slurm </div></div> <div class="language-select"><p data-svelte-h="svelte-6k9yf4">Set <code>terminationGracePeriodSeconds</code> in your Pod or Job spec. The exact field location varies by trainer (Kubeflow Training Operator, Ray, etc.).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-yaml "><!-- HTML_TAG_START --><span class="hljs-attr">spec:</span>
	<span class="hljs-attr">template:</span>
	<span class="hljs-attr">spec:</span>
	<span class="hljs-attr">terminationGracePeriodSeconds:</span> <span class="hljs-number">300</span><!-- HTML_TAG_END --></pre></div> </div> <p data-svelte-h="svelte-19xyat9">Calculate the required grace period as the longest possible training step time plus the checkpoint saving time, plus the 3 second <code>kill_wait</code> delay before the checkpoint begins. For example, if a training step takes up to 2 minutes and saving a checkpoint takes 2 minutes, set at least 243 seconds of grace time.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/trainer_recipes.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_1gaarxw = {
	assets: "/docs/transformers/pr_43265/en",
	base: "/docs/transformers/pr_43265/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/transformers/pr_43265/en/_app/immutable/entry/start.1dae3980.js"),
	import("/docs/transformers/pr_43265/en/_app/immutable/entry/app.66dba7a3.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 713],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 57.2 kB
Xet hash:: 1d065edaedd0dc5026dc075562fe57270a12a3927638702b8ac506901db0c1b1

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.