accelerate/pr_4042/en/_app/immutable/nodes/39.c8a0fbcf.js

import{s as ne,n as ce,o as re}from"../chunks/scheduler.b9285784.js";import{S as oe,i as ie,e as c,s,c as _,h as pe,a as r,d as a,b as n,f as te,g as B,j as J,k as ae,l as Me,m as l,n as g,t as T,o as G,p as k}from"../chunks/index.26bc89a1.js";import{C as de,H as le,E as me}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.ad7e8229.js";import{C as se}from"../chunks/CodeBlock.2768076e.js";function he(Q){let o,v,Z,Y,i,C,p,R,M,L=`When training a PyTorch model with Accelerate, you may often want to save and continue a state of training. Doing so requires
saving and loading the model, optimizer, RNG generators, and the GradScaler. Inside Accelerate are two convenience functions to achieve this quickly:`,I,d,K='<li>Use <a href="/docs/accelerate/pr_4042/en/package_reference/accelerator#accelerate.Accelerator.save_state">save_state()</a> for saving everything mentioned above to a folder location</li> <li>Use <a href="/docs/accelerate/pr_4042/en/package_reference/accelerator#accelerate.Accelerator.load_state">load_state()</a> for loading everything stored from an earlier <code>save_state</code></li>',A,m,P=`To further customize where and how states are saved through <a href="/docs/accelerate/pr_4042/en/package_reference/accelerator#accelerate.Accelerator.save_state">save_state()</a> the <a href="/docs/accelerate/pr_4042/en/package_reference/utilities#accelerate.utils.ProjectConfiguration">ProjectConfiguration</a> class can be used. For example
if <code>automatic_checkpoint_naming</code> is enabled each saved checkpoint will be located then at <code>Accelerator.project_dir/checkpoints/checkpoint_{checkpoint_number}</code>.`,W,h,q="It should be noted that the expectation is that those states come from the same training script, they should not be from two separate scripts.",F,y,D=`<li>By using <a href="/docs/accelerate/pr_4042/en/package_reference/accelerator#accelerate.Accelerator.register_for_checkpointing">register_for_checkpointing()</a>, you can register custom objects to be automatically stored or loaded from the two prior functions,
so long as the object has a <code>state_dict</code> <strong>and</strong> a <code>load_state_dict</code> functionality. This could include objects such as a learning rate scheduler.</li>`,$,j,O="Below is a brief example using checkpointing to save and reload a state during training:",V,f,E,u,S,w,ee=`After resuming from a checkpoint, it may also be desirable to resume from a particular point in the active <code>DataLoader</code> if
the state was saved during the middle of an epoch. You can use <a href="/docs/accelerate/pr_4042/en/package_reference/accelerator#accelerate.Accelerator.skip_first_batches">skip_first_batches()</a> to do so.`,N,U,z,b,H,X,x;return i=new de({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),p=new le({props:{title:"Checkpointing",local:"checkpointing",headingTag:"h1"}}),f=new se({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWltcG9ydCUyMHRvcmNoJTBBJTBBYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihwcm9qZWN0X2RpciUzRCUyMm15JTJGc2F2ZSUyRnBhdGglMjIpJTBBJTBBbXlfc2NoZWR1bGVyJTIwJTNEJTIwdG9yY2gub3B0aW0ubHJfc2NoZWR1bGVyLlN0ZXBMUihteV9vcHRpbWl6ZXIlMkMlMjBzdGVwX3NpemUlM0QxJTJDJTIwZ2FtbWElM0QwLjk5KSUwQW15X21vZGVsJTJDJTIwbXlfb3B0aW1pemVyJTJDJTIwbXlfdHJhaW5pbmdfZGF0YWxvYWRlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUobXlfbW9kZWwlMkMlMjBteV9vcHRpbWl6ZXIlMkMlMjBteV90cmFpbmluZ19kYXRhbG9hZGVyKSUwQSUwQSUyMyUyMFJlZ2lzdGVyJTIwdGhlJTIwTFIlMjBzY2hlZHVsZXIlMEFhY2NlbGVyYXRvci5yZWdpc3Rlcl9mb3JfY2hlY2twb2ludGluZyhteV9zY2hlZHVsZXIpJTBBJTBBJTIzJTIwU2F2ZSUyMHRoZSUyMHN0YXJ0aW5nJTIwc3RhdGUlMEFhY2NlbGVyYXRvci5zYXZlX3N0YXRlKCklMEElMEFkZXZpY2UlMjAlM0QlMjBhY2NlbGVyYXRvci5kZXZpY2UlMEFteV9tb2RlbC50byhkZXZpY2UpJTBBJTBBJTIzJTIwUGVyZm9ybSUyMHRyYWluaW5nJTBBZm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG51bV9lcG9jaHMpJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwYmF0Y2glMjBpbiUyMG15X3RyYWluaW5nX2RhdGFsb2FkZXIlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBteV9vcHRpbWl6ZXIuemVyb19ncmFkKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dHMlMkMlMjB0YXJnZXRzJTIwJTNEJTIwYmF0Y2glMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dHMlMjAlM0QlMjBpbnB1dHMudG8oZGV2aWNlKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRhcmdldHMlMjAlM0QlMjB0YXJnZXRzLnRvKGRldmljZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbXlfbW9kZWwoaW5wdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBteV9sb3NzX2Z1bmN0aW9uKG91dHB1dHMlMkMlMjB0YXJnZXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbXlfb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMG15X3NjaGVkdWxlci5zdGVwKCklMEElMEElMjMlMjBSZXN0b3JlJTIwdGhlJTIwcHJldmlvdXMlMjBzdGF0ZSUwQWFjY2VsZXJhdG9yLmxvYWRfc3RhdGUoJTIybXklMkZzYXZlJTJGcGF0aCUyRmNoZWNrcG9pbnRpbmclMkZjaGVja3BvaW50XzAlMjIp",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator
<span class="hljs-keyword">import</span> torch

accelerator = Accelerator(project_dir=<span class="hljs-string">&quot;my/save/path&quot;</span>)

my_scheduler = torch.optim.lr_scheduler.StepLR(my_optimizer, step_size=<span class="hljs-number">1</span>, gamma=<span class="hljs-number">0.99</span>)
my_model, my_optimizer, my_training_dataloader = accelerator.prepare(my_model, my_optimizer, my_training_dataloader)

<span class="hljs-comment"># Register the LR scheduler</span>
accelerator.register_for_checkpointing(my_scheduler)

<span class="hljs-comment"># Save the starting state</span>
accelerator.save_state()

device = accelerator.device
my_model.to(device)

<span class="hljs-comment"># Perform training</span>
<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_epochs):
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> my_training_dataloader:
        my_optimizer.zero_grad()
        inputs, targets = batch
        inputs = inputs.to(device)
        targets = targets.to(device)
        outputs = my_model(inputs)
        loss = my_loss_function(outputs, targets)
        accelerator.backward(loss)
        my_optimizer.step()
    my_scheduler.step()

<span class="hljs-comment"># Restore the previous state</span>
accelerator.load_state(<span class="hljs-string">&quot;my/save/path/checkpointing/checkpoint_0&quot;</span>)`,lang:"python",wrap:!1}}),u=new le({props:{title:"Restoring the state of the DataLoader",local:"restoring-the-state-of-the-dataloader",headingTag:"h2"}}),U=new se({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IocHJvamVjdF9kaXIlM0QlMjJteSUyRnNhdmUlMkZwYXRoJTIyKSUwQSUwQXRyYWluX2RhdGFsb2FkZXIlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKHRyYWluX2RhdGFsb2FkZXIpJTBBYWNjZWxlcmF0b3IubG9hZF9zdGF0ZSglMjJteV9zdGF0ZSUyMiklMEElMEElMjMlMjBBc3N1bWUlMjB0aGUlMjBjaGVja3BvaW50JTIwd2FzJTIwc2F2ZWQlMjAxMDAlMjBzdGVwcyUyMGludG8lMjB0aGUlMjBlcG9jaCUwQXNraXBwZWRfZGF0YWxvYWRlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnNraXBfZmlyc3RfYmF0Y2hlcyh0cmFpbl9kYXRhbG9hZGVyJTJDJTIwMTAwKSUwQSUwQSUyMyUyMEFmdGVyJTIwdGhlJTIwZmlyc3QlMjBpdGVyYXRpb24lMkMlMjBnbyUyMGJhY2slMjB0byUyMCU2MHRyYWluX2RhdGFsb2FkZXIlNjAlMEElMEElMjMlMjBGaXJzdCUyMGVwb2NoJTBBZm9yJTIwYmF0Y2glMjBpbiUyMHNraXBwZWRfZGF0YWxvYWRlciUzQSUwQSUyMCUyMCUyMCUyMCUyMyUyMERvJTIwc29tZXRoaW5nJTBBJTIwJTIwJTIwJTIwcGFzcyUwQSUwQSUyMyUyMFNlY29uZCUyMGVwb2NoJTBBZm9yJTIwYmF0Y2glMjBpbiUyMHRyYWluX2RhdGFsb2FkZXIlM0ElMEElMjAlMjAlMjAlMjAlMjMlMjBEbyUyMHNvbWV0aGluZyUwQSUyMCUyMCUyMCUyMHBhc3M=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator(project_dir=<span class="hljs-string">&quot;my/save/path&quot;</span>)

train_dataloader = accelerator.prepare(train_dataloader)
accelerator.load_state(<span class="hljs-string">&quot;my_state&quot;</span>)

<span class="hljs-comment"># Assume the checkpoint was saved 100 steps into the epoch</span>
skipped_dataloader = accelerator.skip_first_batches(train_dataloader, <span class="hljs-number">100</span>)

<span class="hljs-comment"># After the first iteration, go back to \`train_dataloader\`</span>

<span class="hljs-comment"># First epoch</span>
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> skipped_dataloader:
    <span class="hljs-comment"># Do something</span>
    <span class="hljs-keyword">pass</span>

<span class="hljs-comment"># Second epoch</span>
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
    <span class="hljs-comment"># Do something</span>
    <span class="hljs-keyword">pass</span>`,lang:"python",wrap:!1}}),b=new me({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/checkpoint.md"}}),{c(){o=c("meta"),v=s(),Z=c("p"),Y=s(),_(i.$$.fragment),C=s(),_(p.$$.fragment),R=s(),M=c("p"),M.textContent=L,I=s(),d=c("ul"),d.innerHTML=K,A=s(),m=c("p"),m.innerHTML=P,W=s(),h=c("p"),h.textContent=q,F=s(),y=c("ul"),y.innerHTML=D,$=s(),j=c("p"),j.textContent=O,V=s(),_(f.$$.fragment),E=s(),_(u.$$.fragment),S=s(),w=c("p"),w.innerHTML=ee,N=s(),_(U.$$.fragment),z=s(),_(b.$$.fragment),H=s(),X=c("p"),this.h()},l(e){const t=pe("svelte-u9bgzb",document.head);o=r(t,"META",{name:!0,content:!0}),t.forEach(a),v=n(e),Z=r(e,"P",{}),te(Z).forEach(a),Y=n(e),B(i.$$.fragment,e),C=n(e),B(p.$$.fragment,e),R=n(e),M=r(e,"P",{"data-svelte-h":!0}),J(M)!=="svelte-13ap35m"&&(M.textContent=L),I=n(e),d=r(e,"UL",{"data-svelte-h":!0}),J(d)!=="svelte-1v15xn9"&&(d.innerHTML=K),A=n(e),m=r(e,"P",{"data-svelte-h":!0}),J(m)!=="svelte-1d0thq2"&&(m.innerHTML=P),W=n(e),h=r(e,"P",{"data-svelte-h":!0}),J(h)!=="svelte-iddkef"&&(h.textContent=q),F=n(e),y=r(e,"UL",{"data-svelte-h":!0}),J(y)!=="svelte-bfdb53"&&(y.innerHTML=D),$=n(e),j=r(e,"P",{"data-svelte-h":!0}),J(j)!=="svelte-3hrue1"&&(j.textContent=O),V=n(e),B(f.$$.fragment,e),E=n(e),B(u.$$.fragment,e),S=n(e),w=r(e,"P",{"data-svelte-h":!0}),J(w)!=="svelte-q8gltn"&&(w.innerHTML=ee),N=n(e),B(U.$$.fragment,e),z=n(e),B(b.$$.fragment,e),H=n(e),X=r(e,"P",{}),te(X).forEach(a),this.h()},h(){ae(o,"name","hf:doc:metadata"),ae(o,"content",ye)},m(e,t){Me(document.head,o),l(e,v,t),l(e,Z,t),l(e,Y,t),g(i,e,t),l(e,C,t),g(p,e,t),l(e,R,t),l(e,M,t),l(e,I,t),l(e,d,t),l(e,A,t),l(e,m,t),l(e,W,t),l(e,h,t),l(e,F,t),l(e,y,t),l(e,$,t),l(e,j,t),l(e,V,t),g(f,e,t),l(e,E,t),g(u,e,t),l(e,S,t),l(e,w,t),l(e,N,t),g(U,e,t),l(e,z,t),g(b,e,t),l(e,H,t),l(e,X,t),x=!0},p:ce,i(e){x||(T(i.$$.fragment,e),T(p.$$.fragment,e),T(f.$$.fragment,e),T(u.$$.fragment,e),T(U.$$.fragment,e),T(b.$$.fragment,e),x=!0)},o(e){G(i.$$.fragment,e),G(p.$$.fragment,e),G(f.$$.fragment,e),G(u.$$.fragment,e),G(U.$$.fragment,e),G(b.$$.fragment,e),x=!1},d(e){e&&(a(v),a(Z),a(Y),a(C),a(R),a(M),a(I),a(d),a(A),a(m),a(W),a(h),a(F),a(y),a($),a(j),a(V),a(E),a(S),a(w),a(N),a(z),a(H),a(X)),a(o),k(i,e),k(p,e),k(f,e),k(u,e),k(U,e),k(b,e)}}}const ye='{"title":"Checkpointing","local":"checkpointing","sections":[{"title":"Restoring the state of the DataLoader","local":"restoring-the-state-of-the-dataloader","sections":[],"depth":2}],"depth":1}';function je(Q){return re(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class be extends oe{constructor(o){super(),ie(this,o,je,he,ne,{})}}export{be as component};