Hugging Face's logo

Buckets:
hf-doc-build
/
doc-dev

Files
xet
hf-doc-build/doc-dev / transformers /pr_35339 /ko /_app /immutable /nodes /161.b4e2d13a.js
rtrm's picture
rtrm
download
raw
79.3 kB
import{s as ja,o as wa,n as A}from"../chunks/scheduler.bdbef820.js";import{S as ya,i as Ua,g as i,s as r,r as J,A as fa,h as c,f as e,c as p,j as ma,u as j,x as o,k as Ta,y as da,a,v as w,d as y,t as U,w as f}from"../chunks/index.33f81d56.js";import{T as Kl}from"../chunks/Tip.34194030.js";import{C as g}from"../chunks/CodeBlock.362b34a4.js";import{H as Z,E as ua}from"../chunks/EditOnGithub.a9246e21.js";import{H as Ja,a as ql}from"../chunks/HfOption.6b792247.js";function ha(h){let t,M='Transformers는 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 클래스 외에도 번역이나 요약과 같은 시퀀스-투-시퀀스 작업을 위한 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Seq2SeqTrainer">Seq2SeqTrainer</a> 클래스도 제공합니다. 또한 <a href="https://hf.co/docs/trl" rel="nofollow">TRL</a> 라이브러리에는 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 클래스를 감싸고 Llama-2 및 Mistral과 같은 언어 모델을 자동 회귀 기법으로 훈련하는 데 최적화된 <code>SFTTrainer</code> 클래스 입니다. <code>SFTTrainer</code>는 시퀀스 패킹, LoRA, 양자화 및 DeepSpeed와 같은 기능을 지원하여 크기 상관없이 모델 효율적으로 확장할 수 있습니다.',n,d,m,u,_='이들 다른 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 유형 클래스에 대해 더 알고 싶다면 <a href="./main_classes/trainer">API 참조</a>를 확인하여 언제 어떤 클래스가 적합할지 얼마든지 확인하세요. 일반적으로 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>는 가장 다재다능한 옵션으로, 다양한 작업에 적합합니다. <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Seq2SeqTrainer">Seq2SeqTrainer</a>는 시퀀스-투-시퀀스 작업을 위해 설계되었고, <code>SFTTrainer</code>는 언어 모델 훈련을 위해 설계되었습니다.';return{c(){t=i("p"),t.innerHTML=M,n=r(),d=i("br"),m=r(),u=i("p"),u.innerHTML=_},l(I){t=c(I,"P",{"data-svelte-h":!0}),o(t)!=="svelte-1pal5ux"&&(t.innerHTML=M),n=p(I),d=c(I,"BR",{}),m=p(I),u=c(I,"P",{"data-svelte-h":!0}),o(u)!=="svelte-12e7w4l"&&(u.innerHTML=_)},m(I,T){a(I,t,T),a(I,n,T),a(I,d,T),a(I,m,T),a(I,u,T)},p:A,d(I){I&&(e(t),e(n),e(d),e(m),e(u))}}}function _a(h){let t,M='로깅 API에 대한 자세한 내용은 <a href="./main_classes/logging">로깅</a> API 레퍼런스를 확인하세요.';return{c(){t=i("p"),t.innerHTML=M},l(n){t=c(n,"P",{"data-svelte-h":!0}),o(t)!=="svelte-1djg64a"&&(t.innerHTML=M)},m(n,d){a(n,t,d)},p:A,d(n){n&&e(t)}}}function ba(h){let t,M='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>는 <code>Trainer.__init__()</code> 메소드에서 각 노드에 대해 로그 레벨을 별도로 설정하므로, 다른 Transformers 기능을 사용할 경우 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 객체를 생성하기 전에 이를 미리 설정하는 것이 좋습니다.';return{c(){t=i("p"),t.innerHTML=M},l(n){t=c(n,"P",{"data-svelte-h":!0}),o(t)!=="svelte-mokbql"&&(t.innerHTML=M)},m(n,d){a(n,t,d)},p:A,d(n){n&&e(t)}}}function Ia(h){let t,M;return t=new g({props:{code:"bXlfYXBwLnB5JTIwLi4uJTIwLS1sb2dfbGV2ZWwlMjB3YXJuaW5nJTIwLS1sb2dfbGV2ZWxfcmVwbGljYSUyMGVycm9y",highlighted:"my_app.py ... --log_level warning --log_level_replica error",wrap:!1}}),{c(){J(t.$$.fragment)},l(n){j(t.$$.fragment,n)},m(n,d){w(t,n,d),M=!0},p:A,i(n){M||(y(t.$$.fragment,n),M=!0)},o(n){U(t.$$.fragment,n),M=!1},d(n){f(t,n)}}}function ga(h){let t,M="멀티 노드 환경에서는 <code>log_on_each_node 0</code> 매개변수를 추가합니다.",n,d,m;return d=new g({props:{code:"bXlfYXBwLnB5JTIwLi4uJTIwLS1sb2dfbGV2ZWwlMjB3YXJuaW5nJTIwLS1sb2dfbGV2ZWxfcmVwbGljYSUyMGVycm9yJTIwLS1sb2dfb25fZWFjaF9ub2RlJTIwMCUwQSUwQSUyMyUyMCVFQyU5OCVBNCVFQiVBNSU5OCVFQiVBNyU4QyUyMCVFQiVCMyVCNCVFQSVCMyVBMCVFRCU5NSU5OCVFQiU4RiU4NCVFQiVBMSU5RCUyMCVFQyU4NCVBNCVFQyVBMCU5NSUwQW15X2FwcC5weSUyMC4uLiUyMC0tbG9nX2xldmVsJTIwZXJyb3IlMjAtLWxvZ19sZXZlbF9yZXBsaWNhJTIwZXJyb3IlMjAtLWxvZ19vbl9lYWNoX25vZGUlMjAw",highlighted:`my_app.py ... --log_level warning --log_level_replica error --log_on_each_node 0
<span class="hljs-comment"># 오류만 보고하도록 설정</span>
my_app.py ... --log_level error --log_level_replica error --log_on_each_node 0`,wrap:!1}}),{c(){t=i("p"),t.innerHTML=M,n=r(),J(d.$$.fragment)},l(u){t=c(u,"P",{"data-svelte-h":!0}),o(t)!=="svelte-1tjciiw"&&(t.innerHTML=M),n=p(u),j(d.$$.fragment,u)},m(u,_){a(u,t,_),a(u,n,_),w(d,u,_),m=!0},p:A,i(u){m||(y(d.$$.fragment,u),m=!0)},o(u){U(d.$$.fragment,u),m=!1},d(u){u&&(e(t),e(n)),f(d,u)}}}function Aa(h){let t,M,n,d;return t=new ql({props:{id:"logging",option:"single node",$$slots:{default:[Ia]},$$scope:{ctx:h}}}),n=new ql({props:{id:"logging",option:"multi-node",$$slots:{default:[ga]},$$scope:{ctx:h}}}),{c(){J(t.$$.fragment),M=r(),J(n.$$.fragment)},l(m){j(t.$$.fragment,m),M=p(m),j(n.$$.fragment,m)},m(m,u){w(t,m,u),a(m,M,u),w(n,m,u),d=!0},p(m,u){const _={};u&2&&(_.$$scope={dirty:u,ctx:m}),t.$set(_);const I={};u&2&&(I.$$scope={dirty:u,ctx:m}),n.$set(I)},i(m){d||(y(t.$$.fragment,m),y(n.$$.fragment,m),d=!0)},o(m){U(t.$$.fragment,m),U(n.$$.fragment,m),d=!1},d(m){m&&e(M),f(t,m),f(n,m)}}}function Ca(h){let t,M="저자에 따르면, <code>grad_norm</code> 없이 <code>AdaLomo</code>를 사용하는 것이 더 나은 성능과 높은 처리량을 제공한다고 합니다.";return{c(){t=i("p"),t.innerHTML=M},l(n){t=c(n,"P",{"data-svelte-h":!0}),o(t)!=="svelte-733wbz"&&(t.innerHTML=M)},m(n,d){a(n,t,d)},p:A,d(n){n&&e(t)}}}function Za(h){let t,M='FSDP 샤딩 전략, CPU 오프로드 및 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>와 함께 사용할 수 있는 더 많은 기능을 알아보려면 <a href="fsdp">Fully Sharded Data Parallel</a> 가이드를 확인하세요.';return{c(){t=i("p"),t.innerHTML=M},l(n){t=c(n,"P",{"data-svelte-h":!0}),o(t)!=="svelte-11kbnod"&&(t.innerHTML=M)},m(n,d){a(n,t,d)},p:A,d(n){n&&e(t)}}}function Va(h){let t,M;return t=new g({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwTVVMVElfR1BVJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTBBZG93bmNhc3RfYmYxNiUzQSUyMCdubyclMEFncHVfaWRzJTNBJTIwYWxsJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUyMCUyMyUyMCVFQiU4NSVCOCVFQiU5MyU5QyVFQyU5NyU5MCUyMCVFQiU5NCVCMCVFQiU5RCVCQyUyMCVFQyU4OCU5QyVFQyU5QyU4NCVFQiVBNSVCQyUyMCVFQiVCMyU4MCVFQSVCMiVCRCVFRCU5NSU5OCVFQyU4NCVCOCVFQyU5QSU5NCUwQW1haW5fcHJvY2Vzc19pcCUzQSUyMDE5Mi4xNjguMjAuMSUwQW1haW5fcHJvY2Vzc19wb3J0JTNBJTIwOTg5OCUwQW1haW5fdHJhaW5pbmdfZnVuY3Rpb24lM0ElMjBtYWluJTBBbWl4ZWRfcHJlY2lzaW9uJTNBJTIwZnAxNiUwQW51bV9tYWNoaW5lcyUzQSUyMDIlMEFudW1fcHJvY2Vzc2VzJTNBJTIwOCUwQXJkenZfYmFja2VuZCUzQSUyMHN0YXRpYyUwQXNhbWVfbmV0d29yayUzQSUyMHRydWUlMEF0cHVfZW52JTNBJTIwJTVCJTVEJTBBdHB1X3VzZV9jbHVzdGVyJTNBJTIwZmFsc2UlMEF0cHVfdXNlX3N1ZG8lM0ElMjBmYWxzZSUwQXVzZV9jcHUlM0ElMjBmYWxzZQ==",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">MULTI_GPU</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">gpu_ids:</span> <span class="hljs-string">all</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span> <span class="hljs-comment"># 노드에 따라 순위를 변경하세요</span>
<span class="hljs-attr">main_process_ip:</span> <span class="hljs-number">192.168</span><span class="hljs-number">.20</span><span class="hljs-number">.1</span>
<span class="hljs-attr">main_process_port:</span> <span class="hljs-number">9898</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">fp16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">8</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){J(t.$$.fragment)},l(n){j(t.$$.fragment,n)},m(n,d){w(t,n,d),M=!0},p:A,i(n){M||(y(t.$$.fragment,n),M=!0)},o(n){U(t.$$.fragment,n),M=!1},d(n){f(t,n)}}}function $a(h){let t,M;return t=new g({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwRlNEUCUwQWRvd25jYXN0X2JmMTYlM0ElMjAnbm8nJTBBZnNkcF9jb25maWclM0ElMEElMjAlMjBmc2RwX2F1dG9fd3JhcF9wb2xpY3klM0ElMjBUUkFOU0ZPUk1FUl9CQVNFRF9XUkFQJTBBJTIwJTIwZnNkcF9iYWNrd2FyZF9wcmVmZXRjaF9wb2xpY3klM0ElMjBCQUNLV0FSRF9QUkUlMEElMjAlMjBmc2RwX2ZvcndhcmRfcHJlZmV0Y2glM0ElMjB0cnVlJTBBJTIwJTIwZnNkcF9vZmZsb2FkX3BhcmFtcyUzQSUyMGZhbHNlJTBBJTIwJTIwZnNkcF9zaGFyZGluZ19zdHJhdGVneSUzQSUyMDElMEElMjAlMjBmc2RwX3N0YXRlX2RpY3RfdHlwZSUzQSUyMEZVTExfU1RBVEVfRElDVCUwQSUyMCUyMGZzZHBfc3luY19tb2R1bGVfc3RhdGVzJTNBJTIwdHJ1ZSUwQSUyMCUyMGZzZHBfdHJhbnNmb3JtZXJfbGF5ZXJfY2xzX3RvX3dyYXAlM0ElMjBCZXJ0TGF5ZXIlMEElMjAlMjBmc2RwX3VzZV9vcmlnX3BhcmFtcyUzQSUyMHRydWUlMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBiZjE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjAyJTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">FSDP</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">fsdp_config:</span>
 <span class="hljs-attr">fsdp_auto_wrap_policy:</span> <span class="hljs-string">TRANSFORMER_BASED_WRAP</span>
 <span class="hljs-attr">fsdp_backward_prefetch_policy:</span> <span class="hljs-string">BACKWARD_PRE</span>
 <span class="hljs-attr">fsdp_forward_prefetch:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">fsdp_offload_params:</span> <span class="hljs-literal">false</span>
 <span class="hljs-attr">fsdp_sharding_strategy:</span> <span class="hljs-number">1</span>
 <span class="hljs-attr">fsdp_state_dict_type:</span> <span class="hljs-string">FULL_STATE_DICT</span>
 <span class="hljs-attr">fsdp_sync_module_states:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">fsdp_transformer_layer_cls_to_wrap:</span> <span class="hljs-string">BertLayer</span>
 <span class="hljs-attr">fsdp_use_orig_params:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">bf16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){J(t.$$.fragment)},l(n){j(t.$$.fragment,n)},m(n,d){w(t,n,d),M=!0},p:A,i(n){M||(y(t.$$.fragment,n),M=!0)},o(n){U(t.$$.fragment,n),M=!1},d(n){f(t,n)}}}function Ba(h){let t,M;return t=new g({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwJTIwZGVlcHNwZWVkX2NvbmZpZ19maWxlJTNBJTIwJTJGaG9tZSUyRnVzZXIlMkZjb25maWdzJTJGZHNfemVybzNfY29uZmlnLmpzb24lMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBZGlzdHJpYnV0ZWRfdHlwZSUzQSUyMERFRVBTUEVFRCUwQWRvd25jYXN0X2JmMTYlM0ElMjAnbm8nJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUwQW1haW5fdHJhaW5pbmdfZnVuY3Rpb24lM0ElMjBtYWluJTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjA0JTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">deepspeed_config:</span>
 <span class="hljs-attr">deepspeed_config_file:</span> <span class="hljs-string">/home/user/configs/ds_zero3_config.json</span>
 <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">4</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){J(t.$$.fragment)},l(n){j(t.$$.fragment,n)},m(n,d){w(t,n,d),M=!0},p:A,i(n){M||(y(t.$$.fragment,n),M=!0)},o(n){U(t.$$.fragment,n),M=!1},d(n){f(t,n)}}}function Xa(h){let t,M;return t=new g({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTBBJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNBJTIwMSUwQSUyMCUyMGdyYWRpZW50X2NsaXBwaW5nJTNBJTIwMC43JTBBJTIwJTIwb2ZmbG9hZF9vcHRpbWl6ZXJfZGV2aWNlJTNBJTIwY3B1JTBBJTIwJTIwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlM0ElMjBjcHUlMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBJTIwJTIwemVyb19zdGFnZSUzQSUyMDIlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwREVFUFNQRUVEJTBBZG93bmNhc3RfYmYxNiUzQSUyMCdubyclMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBiZjE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjA0JTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">deepspeed_config:</span>
<span class="hljs-attr">gradient_accumulation_steps:</span> <span class="hljs-number">1</span>
 <span class="hljs-attr">gradient_clipping:</span> <span class="hljs-number">0.7</span>
 <span class="hljs-attr">offload_optimizer_device:</span> <span class="hljs-string">cpu</span>
 <span class="hljs-attr">offload_param_device:</span> <span class="hljs-string">cpu</span>
 <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
 <span class="hljs-attr">zero_stage:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">bf16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">4</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_env:</span> []
<span class="hljs-attr">tpu_use_cluster:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">tpu_use_sudo:</span> <span class="hljs-literal">false</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),{c(){J(t.$$.fragment)},l(n){j(t.$$.fragment,n)},m(n,d){w(t,n,d),M=!0},p:A,i(n){M||(y(t.$$.fragment,n),M=!0)},o(n){U(t.$$.fragment,n),M=!1},d(n){f(t,n)}}}function Qa(h){let t,M,n,d,m,u,_,I;return t=new ql({props:{id:"config",option:"DistributedDataParallel",$$slots:{default:[Va]},$$scope:{ctx:h}}}),n=new ql({props:{id:"config",option:"FSDP",$$slots:{default:[$a]},$$scope:{ctx:h}}}),m=new ql({props:{id:"config",option:"DeepSpeed",$$slots:{default:[Ba]},$$scope:{ctx:h}}}),_=new ql({props:{id:"config",option:"DeepSpeed with Accelerate plugin",$$slots:{default:[Xa]},$$scope:{ctx:h}}}),{c(){J(t.$$.fragment),M=r(),J(n.$$.fragment),d=r(),J(m.$$.fragment),u=r(),J(_.$$.fragment)},l(T){j(t.$$.fragment,T),M=p(T),j(n.$$.fragment,T),d=p(T),j(m.$$.fragment,T),u=p(T),j(_.$$.fragment,T)},m(T,b){w(t,T,b),a(T,M,b),w(n,T,b),a(T,d,b),w(m,T,b),a(T,u,b),w(_,T,b),I=!0},p(T,b){const F={};b&2&&(F.$$scope={dirty:b,ctx:T}),t.$set(F);const C={};b&2&&(C.$$scope={dirty:b,ctx:T}),n.$set(C);const Pl={};b&2&&(Pl.$$scope={dirty:b,ctx:T}),m.$set(Pl);const E={};b&2&&(E.$$scope={dirty:b,ctx:T}),_.$set(E)},i(T){I||(y(t.$$.fragment,T),y(n.$$.fragment,T),y(m.$$.fragment,T),y(_.$$.fragment,T),I=!0)},o(T){U(t.$$.fragment,T),U(n.$$.fragment,T),U(m.$$.fragment,T),U(_.$$.fragment,T),I=!1},d(T){T&&(e(M),e(d),e(u)),f(t,T),f(n,T),f(m,T),f(_,T)}}}function Ra(h){let t,M,n,d,m,u,_,I='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>는 Transformers 라이브러리에 구현된 PyTorch 모델을 반복하여 훈련 및 평가 과정입니다. 훈련에 필요한 요소(모델, 토크나이저, 데이터셋, 평가 함수, 훈련 하이퍼파라미터 등)만 제공하면 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>가 필요한 나머지 작업을 처리합니다. 이를 통해 직접 훈련 루프를 작성하지 않고도 빠르게 훈련을 시작할 수 있습니다. 또한 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>는 강력한 맞춤 설정과 다양한 훈련 옵션을 제공하여 사용자 맞춤 훈련이 가능합니다.',T,b,F,C,Pl='시작하기 전에, 분산 환경에서 PyTorch 훈련과 실행을 할 수 있게 <a href="https://hf.co/docs/accelerate" rel="nofollow">Accelerate</a> 라이브러리가 설치되었는지 확인하세요.',E,W,ls,k,fe='이 가이드는 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 클래스에 대한 개요를 제공합니다.',ss,N,es,G,de='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>는 기본적인 훈련 루프에 필요한 모든 코드를 포함하고 있습니다.',as,v,ue="<li>손실을 계산하는 훈련 단계를 수행합니다.</li> <li><code>backward</code> 메소드로 그레이디언트를 계산합니다.</li> <li>그레이디언트를 기반으로 가중치를 업데이트합니다.</li> <li>정해진 에폭 수에 도달할 때까지 이 과정을 반복합니다.</li>",ts,S,he='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 클래스는 PyTorch와 훈련 과정에 익숙하지 않거나 막 시작한 경우에도 훈련이 가능하도록 필요한 모든 코드를 추상화하였습니다. 또한 매번 훈련 루프를 손수 작성하지 않아도 되며, 훈련에 필요한 모델과 데이터셋 같은 필수 구성 요소만 제공하면, [Trainer] 클래스가 나머지를 처리합니다.',ns,Y,_e='훈련 옵션이나 하이퍼파라미터를 지정하려면, <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a> 클래스에서 확인 할 수 있습니다. 예를 들어, 모델을 저장할 디렉토리를 <code>output_dir</code>에 정의하고, 훈련 후에 Hub로 모델을 푸시하려면 <code>push_to_hub=True</code>로 설정합니다.',rs,H,ps,z,be='<code>training_args</code>를 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>에 모델, 데이터셋, 데이터셋 전처리 도구(데이터 유형에 따라 토크나이저, 특징 추출기 또는 이미지 프로세서일 수 있음), 데이터 수집기 및 훈련 중 확인할 지표를 계산할 함수를 함께 전달하세요.',Ms,x,Ie='마지막으로, <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.train">train()</a>를 호출하여 훈련을 시작하세요!',is,L,cs,D,os,q,ge='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 클래스는 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>의 <code>output_dir</code> 매개변수에 지정된 디렉토리에 모델 체크포인트를 저장합니다. 체크포인트는 <code>checkpoint-000</code> 하위 폴더에 저장되며, 여기서 끝의 숫자는 훈련 단계에 해당합니다. 체크포인트를 저장하면 나중에 훈련을 재개할 때 유용합니다.',ms,P,Ts,O,Ae='체크포인트를 Hub에 푸시하려면 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>에서 <code>push_to_hub=True</code>로 설정하여 커밋하고 푸시할 수 있습니다. 체크포인트 저장 방법을 결정하는 다른 옵션은 <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.hub_strategy" rel="nofollow"><code>hub_strategy</code></a> 매개변수에서 설정합니다:',Js,K,Ce="<li><code>hub_strategy=&quot;checkpoint&quot;</code>는 최신 체크포인트를 “last-checkpoint”라는 하위 폴더에 푸시하여 훈련을 재개할 수 있습니다.</li> <li><code>hub_strategy=&quot;all_checkpoints&quot;</code>는 모든 체크포인트를 <code>output_dir</code>에 정의된 디렉토리에 푸시합니다(모델 리포지토리에서 폴더당 하나의 체크포인트를 볼 수 있습니다).</li>",js,ll,Ze='체크포인트에서 훈련을 재개할 때, <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>는 체크포인트가 저장될 때와 동일한 Python, NumPy 및 PyTorch RNG 상태를 유지하려고 합니다. 하지만 PyTorch는 기본 설정으로 ‘일관된 결과를 보장하지 않음’으로 많이 되어있기 때문에, RNG 상태가 동일할 것이라고 보장할 수 없습니다. 따라서, 일관된 결과가 보장되도록 활성화 하려면, <a href="https://pytorch.org/docs/stable/notes/randomness#controlling-sources-of-randomness" rel="nofollow">랜덤성 제어</a> 가이드를 참고하여 훈련을 완전히 일관된 결과를 보장 받도록 만들기 위해 활성화할 수 있는 항목을 확인하세요. 다만, 특정 설정을 결정적으로 만들면 훈련이 느려질 수 있습니다.',ws,sl,ys,el,Ve='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 클래스는 접근성과 용이성을 염두에 두고 설계되었지만, 더 다양한 기능을 원하는 사용자들을 위해 다양한 맞춤 설정 옵션을 제공합니다. <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>의 많은 메소드는 서브클래스화 및 오버라이드하여 원하는 기능을 제공할 수 있으며, 이를 통해 전체 훈련 루프를 다시 작성할 필요 없이 원하는 기능을 추가할 수 있습니다. 이러한 메소드에는 다음이 포함됩니다:',Us,al,$e='<li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.get_train_dataloader">get_train_dataloader()</a>는 훈련 데이터로더를 생성합니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.get_eval_dataloader">get_eval_dataloader()</a>는 평가 데이터로더를 생성합니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.get_test_dataloader">get_test_dataloader()</a>는 테스트 데이터로더를 생성합니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.log">log()</a>는 훈련을 모니터링하는 다양한 객체에 대한 정보를 로그로 남깁니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.create_optimizer_and_scheduler">create_optimizer_and_scheduler()</a>는 <code>__init__</code>에서 전달되지 않은 경우 옵티마이저와 학습률 스케줄러를 생성합니다. 이들은 각각 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.create_optimizer">create_optimizer()</a> 및 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.create_scheduler">create_scheduler()</a>로 별도로 맞춤 설정 할 수 있습니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.compute_loss">compute_loss()</a>는 훈련 입력 배치에 대한 손실을 계산합니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.training_step">training_step()</a>는 훈련 단계를 수행합니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.prediction_step">prediction_step()</a>는 예측 및 테스트 단계를 수행합니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.evaluate">evaluate()</a>는 모델을 평가하고 평가 지표을 반환합니다.</li> <li><a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.predict">predict()</a>는 테스트 세트에 대한 예측(레이블이 있는 경우 지표 포함)을 수행합니다.</li>',fs,tl,Be='예를 들어, <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.compute_loss">compute_loss()</a> 메소드를 맞춤 설정하여 가중 손실을 사용하려는 경우:',ds,nl,us,rl,hs,pl,Xe='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>를 맞춤 설정하는 또 다른 방법은 <a href="callbacks">콜백</a>을 사용하는 것입니다. 콜백은 훈련 루프에서 <em>변화를 주지 않습니다</em>. 훈련 루프의 상태를 검사한 후 상태에 따라 일부 작업(조기 종료, 결과 로그 등)을 실행합니다. 즉, 콜백은 사용자 정의 손실 함수와 같은 것을 구현하는 데 사용할 수 없으며, 이를 위해서는 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer.compute_loss">compute_loss()</a> 메소드를 서브클래스화하고 오버라이드해야 합니다.',_s,Ml,Qe="예를 들어, 훈련 루프에 10단계 후 조기 종료 콜백을 추가하려면 다음과 같이 합니다.",bs,il,Is,cl,Re='그런 다음, 이를 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>의 <code>callback</code> 매개변수에 전달합니다.',gs,ol,As,ml,Cs,V,Zs,Tl,Fe='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>는 기본적으로 <code>logging.INFO</code>로 설정되어 있어 오류, 경고 및 기타 기본 정보를 보고합니다. 분산 환경에서는 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 복제본이 <code>logging.WARNING</code>으로 설정되어 오류와 경고만 보고합니다. <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>의 <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.log_level" rel="nofollow"><code>log_level</code></a> 및 <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.log_level_replica" rel="nofollow"><code>log_level_replica</code></a> 매개변수로 로그 레벨을 변경할 수 있습니다.',Vs,Jl,Ee='각 노드의 로그 레벨 설정을 구성하려면 <a href="https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments.log_on_each_node" rel="nofollow"><code>log_on_each_node</code></a> 매개변수를 사용하여 각 노드에서 로그 레벨을 사용할지 아니면 주 노드에서만 사용할지 결정하세요.',$s,$,Bs,jl,We="예를 들어, 메인 코드와 모듈을 각 노드에 따라 동일한 로그 레벨을 사용하도록 설정하려면 다음과 같이 합니다.",Xs,wl,Qs,yl,ke="각 노드에서 기록될 내용을 구성하기 위해 <code>log_level</code>과 <code>log_level_replica</code>를 다양한 조합으로 사용해보세요.",Rs,B,Fs,Ul,Es,fl,Ne='<a href="https://hf.co/papers/2310.05914" rel="nofollow">NEFTune</a>은 훈련 중 임베딩 벡터에 노이즈를 추가하여 성능을 향상시킬 수 있는 기술입니다. <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>에서 이를 활성화하려면 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>의 <code>neftune_noise_alpha</code> 매개변수를 설정하여 노이즈의 양을 조절합니다.',Ws,dl,ks,ul,Ge="NEFTune은 예상치 못한 동작을 피할 목적으로 처음 임베딩 레이어로 복원하기 위해 훈련 후 비활성화 됩니다.",Ns,hl,Gs,_l,ve="Gradient Low-Rank Projection (GaLore)은 전체 매개변수를 학습하면서도 LoRA와 같은 일반적인 저계수 적응 방법보다 더 메모리 효율적인 저계수 학습 전략입니다.",vs,bl,Se="먼저 GaLore 공식 리포지토리를 설치합니다:",Ss,Il,Ys,gl,Ye="그런 다음 <code>optim</code>에 <code>[&quot;galore_adamw&quot;, &quot;galore_adafactor&quot;, &quot;galore_adamw_8bit&quot;]</code> 중 하나와 함께 <code>optim_target_modules</code>를 추가합니다. 이는 적용하려는 대상 모듈 이름에 해당하는 문자열, 정규 표현식 또는 전체 경로의 목록일 수 있습니다. 아래는 end-to-end 예제 스크립트입니다(필요한 경우 <code>pip install trl datasets</code>를 실행):",Hs,Al,zs,Cl,He="GaLore가 지원하는 추가 매개변수를 전달하려면 <code>optim_args</code>를 설정합니다. 예를 들어:",xs,Zl,Ls,Vl,ze='해당 방법에 대한 자세한 내용은 <a href="https://github.com/jiaweizzhao/GaLore" rel="nofollow">원본 리포지토리</a> 또는 <a href="https://arxiv.org/abs/2403.03507" rel="nofollow">논문</a>을 참고하세요.',Ds,$l,xe="현재 GaLore 레이어로 간주되는 Linear 레이어만 훈련 할수 있으며, 저계수 분해를 사용하여 훈련되고 나머지 레이어는 기존 방식으로 최적화됩니다.",qs,Bl,Le="훈련 시작 전에 시간이 약간 걸릴 수 있습니다(NVIDIA A100에서 2B 모델의 경우 약 3분), 하지만 이후 훈련은 원활하게 진행됩니다.",Ps,Xl,De="다음과 같이 옵티마이저 이름에 <code>layerwise</code>를 추가하여 레이어별 최적화를 수행할 수도 있습니다:",Os,Ql,Ks,Rl,qe='레이어별 최적화는 다소 실험적이며 DDP(분산 데이터 병렬)를 지원하지 않으므로, 단일 GPU에서만 훈련 스크립트를 실행할 수 있습니다. 자세한 내용은 <a href="https://github.com/jiaweizzhao/GaLore?tab=readme-ov-file#train-7b-model-with-a-single-gpu-with-24gb-memory" rel="nofollow">이 문서를</a>을 참조하세요. gradient clipping, DeepSpeed 등 다른 기능은 기본적으로 지원되지 않을 수 있습니다. 이러한 문제가 발생하면 <a href="https://github.com/huggingface/transformers/issues" rel="nofollow">GitHub에 이슈를 올려주세요</a>.',le,Fl,se,El,Pe=`LOMO 옵티마이저는 <a href="https://hf.co/papers/2306.09782" rel="nofollow">제한된 자원으로 대형 언어 모델의 전체 매개변수 미세 조정</a>과 <a href="https://hf.co/papers/2310.10195" rel="nofollow">적응형 학습률을 통한 저메모리 최적화(AdaLomo)</a>에서 도입되었습니다.
이들은 모두 효율적인 전체 매개변수 미세 조정 방법으로 구성되어 있습니다. 이러한 옵티마이저들은 메모리 사용량을 줄이기 위해 그레이디언트 계산과 매개변수 업데이트를 하나의 단계로 융합합니다. LOMO에서 지원되는 옵티마이저는 <code>&quot;lomo&quot;</code>와 <code>&quot;adalomo&quot;</code>입니다. 먼저 pypi에서 <code>pip install lomo-optim</code>를 통해 <code>lomo</code>를 설치하거나, GitHub 소스에서 <code>pip install git+https://github.com/OpenLMLab/LOMO.git</code>로 설치하세요.`,ee,X,ae,Wl,Oe='다음은 IMDB 데이터셋에서 <a href="https://huggingface.co/google/gemma-2b" rel="nofollow">google/gemma-2b</a>를 최대 정밀도로 미세 조정하는 간단한 스크립트입니다:',te,kl,ne,Nl,re,Gl,Ke='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a> 클래스는 <a href="https://hf.co/docs/accelerate" rel="nofollow">Accelerate</a>로 구동되며, 이는 <a href="https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api/" rel="nofollow">FullyShardedDataParallel (FSDP)</a> 및 <a href="https://www.deepspeed.ai/" rel="nofollow">DeepSpeed</a>와 같은 통합을 지원하는 분산 환경에서 PyTorch 모델을 쉽게 훈련할 수 있는 라이브러리입니다.',pe,Q,Me,vl,la='<a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>와 Accelerate를 사용하려면 <a href="https://huggingface.co/docs/accelerate/package_reference/cli#accelerate-config" rel="nofollow"><code>accelerate.config</code></a> 명령을 실행하여 훈련 환경을 설정하세요. 이 명령은 훈련 스크립트를 실행할 때 사용할 <code>config_file.yaml</code>을 생성합니다. 예를 들어, 다음 예시는 설정할 수 있는 일부 구성 예입니다.',ie,R,ce,Sl,sa='<a href="https://huggingface.co/docs/accelerate/package_reference/cli#accelerate-launch" rel="nofollow"><code>accelerate_launch</code></a> 명령은 Accelerate와 <a href="/docs/transformers/pr_35339/ko/main_classes/trainer#transformers.Trainer">Trainer</a>를 사용하여 분산 시스템에서 훈련 스크립트를 실행하는 권장 방법이며, <code>config_file.yaml</code>에 지정된 매개변수를 사용합니다. 이 파일은 Accelerate 캐시 폴더에 저장되며 <code>accelerate_launch</code>를 실행할 때 자동으로 로드됩니다.',oe,Yl,ea='예를 들어, FSDP 구성을 사용하여 <a href="https://github.com/huggingface/transformers/blob/f4db565b695582891e43a5e042e5d318e28f20b8/examples/pytorch/text-classification/run_glue.py#L4" rel="nofollow">run_glue.py</a> 훈련 스크립트를 실행하려면 다음과 같이 합니다:',me,Hl,Te,zl,aa="<code>config_file.yaml</code> 파일의 매개변수를 직접 지정할 수도 있습니다:",Je,xl,je,Ll,ta='<code>accelerate_launch</code>와 사용자 정의 구성에 대해 더 알아보려면 <a href="https://huggingface.co/docs/accelerate/basic_tutorials/launch" rel="nofollow">Accelerate 스크립트 실행</a> 튜토리얼을 확인하세요.',we,Dl,ye,Ol,Ue;return m=new Z({props:{title:"Trainer",local:"trainer",headingTag:"h1"}}),b=new Kl({props:{$$slots:{default:[ha]},$$scope:{ctx:h}}}),W=new g({props:{code:"cGlwJTIwaW5zdGFsbCUyMGFjY2VsZXJhdGUlMEElMEElMjMlMjAlRUMlOTclODUlRUElQjclQjglRUIlQTAlODglRUMlOUQlQjQlRUIlOTMlOUMlMEFwaXAlMjBpbnN0YWxsJTIwYWNjZWxlcmF0ZSUyMC0tdXBncmFkZQ==",highlighted:`pip install accelerate
<span class="hljs-comment"># 업그레이드</span>
pip install accelerate --upgrade`,wrap:!1}}),N=new Z({props:{title:"기본 사용법",local:"basic-usage",headingTag:"h2"}}),H=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTBBJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJ5b3VyLW1vZGVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwbGVhcm5pbmdfcmF0ZSUzRDJlLTUlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfZXZhbF9iYXRjaF9zaXplJTNEMTYlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMiUyQyUwQSUyMCUyMCUyMCUyMHdlaWdodF9kZWNheSUzRDAuMDElMkMlMEElMjAlMjAlMjAlMjBldmFsX3N0cmF0ZWd5JTNEJTIyZXBvY2glMjIlMkMlMEElMjAlMjAlMjAlMjBzYXZlX3N0cmF0ZWd5JTNEJTIyZXBvY2glMjIlMkMlMEElMjAlMjAlMjAlMjBsb2FkX2Jlc3RfbW9kZWxfYXRfZW5kJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHB1c2hfdG9faHViJTNEVHJ1ZSUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments
training_args = TrainingArguments(
 output_dir=<span class="hljs-string">&quot;your-model&quot;</span>,
 learning_rate=<span class="hljs-number">2e-5</span>,
 per_device_train_batch_size=<span class="hljs-number">16</span>,
 per_device_eval_batch_size=<span class="hljs-number">16</span>,
 num_train_epochs=<span class="hljs-number">2</span>,
 weight_decay=<span class="hljs-number">0.01</span>,
 eval_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
 save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
 load_best_model_at_end=<span class="hljs-literal">True</span>,
 push_to_hub=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),L=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfbWV0cmljcyUzRGNvbXB1dGVfbWV0cmljcyUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer
trainer = Trainer(
 model=model,
 args=training_args,
 train_dataset=dataset[<span class="hljs-string">&quot;train&quot;</span>],
 eval_dataset=dataset[<span class="hljs-string">&quot;test&quot;</span>],
 tokenizer=tokenizer,
 data_collator=data_collator,
 compute_metrics=compute_metrics,
)
trainer.train()`,wrap:!1}}),D=new Z({props:{title:"체크포인트",local:"checkpoints",headingTag:"h3"}}),P=new g({props:{code:"JTIzJTIwJUVDJUI1JTlDJUVDJThCJUEwJTIwJUVDJUIyJUI0JUVEJTgxJUFDJUVEJThGJUFDJUVDJTlEJUI4JUVEJThBJUI4JUVDJTk3JTkwJUVDJTg0JTlDJTIwJUVDJTlFJUFDJUVBJUIwJTlDJTBBdHJhaW5lci50cmFpbihyZXN1bWVfZnJvbV9jaGVja3BvaW50JTNEVHJ1ZSklMEElMEElMjMlMjAlRUMlQjYlOUMlRUIlQTAlQTUlMjAlRUIlOTQlOTQlRUIlQTAlODklRUQlODYlQTAlRUIlQTYlQUMlRUMlOTclOTAlMjAlRUMlQTAlODAlRUMlOUUlQTUlRUIlOTAlOUMlMjAlRUQlOEElQjklRUMlQTAlOTUlMjAlRUMlQjIlQjQlRUQlODElQUMlRUQlOEYlQUMlRUMlOUQlQjglRUQlOEElQjglRUMlOTclOTAlRUMlODQlOUMlMjAlRUMlOUUlQUMlRUElQjAlOUMlMEF0cmFpbmVyLnRyYWluKHJlc3VtZV9mcm9tX2NoZWNrcG9pbnQlM0QlMjJ5b3VyLW1vZGVsJTJGY2hlY2twb2ludC0xMDAwJTIyKQ==",highlighted:`<span class="hljs-comment"># 최신 체크포인트에서 재개</span>
trainer.train(resume_from_checkpoint=<span class="hljs-literal">True</span>)
<span class="hljs-comment"># 출력 디렉토리에 저장된 특정 체크포인트에서 재개</span>
trainer.train(resume_from_checkpoint=<span class="hljs-string">&quot;your-model/checkpoint-1000&quot;</span>)`,wrap:!1}}),sl=new Z({props:{title:"Trainer 맞춤 설정",local:"customize-the-trainer",headingTag:"h2"}}),nl=new g({props:{code:"ZnJvbSUyMHRvcmNoJTIwaW1wb3J0JTIwbm4lMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVHJhaW5lciUwQSUwQWNsYXNzJTIwQ3VzdG9tVHJhaW5lcihUcmFpbmVyKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGNvbXB1dGVfbG9zcyhzZWxmJTJDJTBBJTBBJTIwbW9kZWwlMkMlMjBpbnB1dHMlMkMlMjByZXR1cm5fb3V0cHV0cyUzREZhbHNlKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVscyUyMCUzRCUyMGlucHV0cy5wb3AoJTIybGFiZWxzJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMCVFQyU4OCU5QyVFQiVCMCVBOSVFRCU5NiVBNSUyMCVFQyVBMCU4NCVFRCU4QyU4QyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBvdXRwdXRzLmdldCglMjJsb2dpdHMlMjIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwJUVDJTg0JTlDJUVCJUExJTlDJTIwJUVCJThCJUE0JUVCJUE1JUI4JTIwJUVBJUIwJTgwJUVDJUE0JTkxJUVDJUI5JTk4JUVCJUExJTlDJTIwMyVFQSVCMCU5QyVFQyU5RCU5OCUyMCVFQiVBMCU4OCVFQyU5RCVCNCVFQiVCOCU5NCVFQyU5NyU5MCUyMCVFQiU4QyU4MCVFRCU5NSU5QyUyMCVFQyU4MiVBQyVFQyU5QSVBOSVFQyU5RSU5MCUyMCVFQyVBMCU5NSVFQyU5RCU5OCUyMCVFQyU4NiU5MCVFQyU4QiVBNCVFQyU5RCU4NCUyMCVFQSVCMyU4NCVFQyU4MiVCMCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3NfZmN0JTIwJTNEJTIwbm4uQ3Jvc3NFbnRyb3B5TG9zcyh3ZWlnaHQlM0R0b3JjaC50ZW5zb3IoJTVCMS4wJTJDJTIwMi4wJTJDJTIwMy4wJTVEJTJDJTIwZGV2aWNlJTNEbW9kZWwuZGV2aWNlKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mY3QobG9naXRzLnZpZXcoLTElMkMlMjBzZWxmLm1vZGVsLmNvbmZpZy5udW1fbGFiZWxzKSUyQyUyMGxhYmVscy52aWV3KC0xKSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAobG9zcyUyQyUyMG91dHB1dHMpJTIwaWYlMjByZXR1cm5fb3V0cHV0cyUyMGVsc2UlMjBsb3Nz",highlighted:`<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer
<span class="hljs-keyword">class</span> <span class="hljs-title class_">CustomTrainer</span>(<span class="hljs-title class_ inherited__">Trainer</span>):
 <span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_loss</span>(<span class="hljs-params">self,
 model, inputs, return_outputs=<span class="hljs-literal">False</span></span>):
 labels = inputs.pop(<span class="hljs-string">&quot;labels&quot;</span>)
 <span class="hljs-comment"># 순방향 전파</span>
 outputs = model(**inputs)
 logits = outputs.get(<span class="hljs-string">&quot;logits&quot;</span>)
 <span class="hljs-comment"># 서로 다른 가중치로 3개의 레이블에 대한 사용자 정의 손실을 계산</span>
 loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([<span class="hljs-number">1.0</span>, <span class="hljs-number">2.0</span>, <span class="hljs-number">3.0</span>], device=model.device))
 loss = loss_fct(logits.view(-<span class="hljs-number">1</span>, self.model.config.num_labels), labels.view(-<span class="hljs-number">1</span>))
 <span class="hljs-keyword">return</span> (loss, outputs) <span class="hljs-keyword">if</span> return_outputs <span class="hljs-keyword">else</span> loss`,wrap:!1}}),rl=new Z({props:{title:"콜백",local:"callbacks",headingTag:"h3"}}),il=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXJDYWxsYmFjayUwQSUwQWNsYXNzJTIwRWFybHlTdG9wcGluZ0NhbGxiYWNrKFRyYWluZXJDYWxsYmFjayklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmJTJDJTIwbnVtX3N0ZXBzJTNEMTApJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5udW1fc3RlcHMlMjAlM0QlMjBudW1fc3RlcHMlMEElMjAlMjAlMjAlMjAlMEElMjAlMjAlMjAlMjBkZWYlMjBvbl9zdGVwX2VuZChzZWxmJTJDJTIwYXJncyUyQyUyMHN0YXRlJTJDJTIwY29udHJvbCUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwc3RhdGUuZ2xvYmFsX3N0ZXAlMjAlM0UlM0QlMjBzZWxmLm51bV9zdGVwcyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMCU3QiUyMnNob3VsZF90cmFpbmluZ19zdG9wJTIyJTNBJTIwVHJ1ZSU3RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlN0Q=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainerCallback
<span class="hljs-keyword">class</span> <span class="hljs-title class_">EarlyStoppingCallback</span>(<span class="hljs-title class_ inherited__">TrainerCallback</span>):
 <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, num_steps=<span class="hljs-number">10</span></span>):
 self.num_steps = num_steps
<span class="hljs-keyword">def</span> <span class="hljs-title function_">on_step_end</span>(<span class="hljs-params">self, args, state, control, **kwargs</span>):
 <span class="hljs-keyword">if</span> state.global_step &gt;= self.num_steps:
 <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;should_training_stop&quot;</span>: <span class="hljs-literal">True</span>}
 <span class="hljs-keyword">else</span>:
 <span class="hljs-keyword">return</span> {}`,wrap:!1}}),ol=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluZXIlMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTBBJTIwJTIwJTIwJTIwYXJncyUzRHRyYWluaW5nX2FyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEZGF0YXNldCU1QiUyMnRlc3QlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2NvbGxhdG9yJTNEZGF0YV9jb2xsYXRvciUyQyUwQSUyMCUyMCUyMCUyMGNvbXB1dGVfbWV0cmljcyUzRGNvbXB1dGVfbWV0cmljcyUyQyUwQSUyMCUyMCUyMCUyMGNhbGxiYWNrcyUzRCU1QkVhcmx5U3RvcHBpbmdDYWxsYmFjaygpJTVEJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer
trainer = Trainer(
 model=model,
 args=training_args,
 train_dataset=dataset[<span class="hljs-string">&quot;train&quot;</span>],
 eval_dataset=dataset[<span class="hljs-string">&quot;test&quot;</span>],
 tokenizer=tokenizer,
 data_collator=data_collator,
 compute_metrics=compute_metrics,
 callbacks=[EarlyStoppingCallback()],
)`,wrap:!1}}),ml=new Z({props:{title:"로깅",local:"logging",headingTag:"h2"}}),V=new Kl({props:{$$slots:{default:[_a]},$$scope:{ctx:h}}}),$=new Kl({props:{$$slots:{default:[ba]},$$scope:{ctx:h}}}),wl=new g({props:{code:"bG9nZ2VyJTIwJTNEJTIwbG9nZ2luZy5nZXRMb2dnZXIoX19uYW1lX18pJTBBJTBBbG9nZ2luZy5iYXNpY0NvbmZpZyglMEElMjAlMjAlMjAlMjBmb3JtYXQlM0QlMjIlMjUoYXNjdGltZSlzJTIwLSUyMCUyNShsZXZlbG5hbWUpcyUyMC0lMjAlMjUobmFtZSlzJTIwLSUyMCUyNShtZXNzYWdlKXMlMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRlZm10JTNEJTIyJTI1bSUyRiUyNWQlMkYlMjVZJTIwJTI1SCUzQSUyNU0lM0ElMjVTJTIyJTJDJTBBJTIwJTIwJTIwJTIwaGFuZGxlcnMlM0QlNUJsb2dnaW5nLlN0cmVhbUhhbmRsZXIoc3lzLnN0ZG91dCklNUQlMkMlMEEpJTBBJTBBbG9nX2xldmVsJTIwJTNEJTIwdHJhaW5pbmdfYXJncy5nZXRfcHJvY2Vzc19sb2dfbGV2ZWwoKSUwQWxvZ2dlci5zZXRMZXZlbChsb2dfbGV2ZWwpJTBBZGF0YXNldHMudXRpbHMubG9nZ2luZy5zZXRfdmVyYm9zaXR5KGxvZ19sZXZlbCklMEF0cmFuc2Zvcm1lcnMudXRpbHMubG9nZ2luZy5zZXRfdmVyYm9zaXR5KGxvZ19sZXZlbCklMEElMEF0cmFpbmVyJTIwJTNEJTIwVHJhaW5lciguLi4p",highlighted:`logger = logging.getLogger(__name__)
logging.basicConfig(
 <span class="hljs-built_in">format</span>=<span class="hljs-string">&quot;%(asctime)s - %(levelname)s - %(name)s - %(message)s&quot;</span>,
 datefmt=<span class="hljs-string">&quot;%m/%d/%Y %H:%M:%S&quot;</span>,
 handlers=[logging.StreamHandler(sys.stdout)],
)
log_level = training_args.get_process_log_level()
logger.setLevel(log_level)
datasets.utils.logging.set_verbosity(log_level)
transformers.utils.logging.set_verbosity(log_level)
trainer = Trainer(...)`,wrap:!1}}),B=new Ja({props:{id:"logging",options:["single node","multi-node"],$$slots:{default:[Aa]},$$scope:{ctx:h}}}),Ul=new Z({props:{title:"NEFTune",local:"neftune",headingTag:"h2"}}),dl=new g({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRyYWluaW5nQXJndW1lbnRzJTJDJTIwVHJhaW5lciUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyguLi4lMkMlMjBuZWZ0dW5lX25vaXNlX2FscGhhJTNEMC4xKSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKC4uLiUyQyUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer
training_args = TrainingArguments(..., neftune_noise_alpha=<span class="hljs-number">0.1</span>)
trainer = Trainer(..., args=training_args)`,wrap:!1}}),hl=new Z({props:{title:"GaLore",local:"galore",headingTag:"h2"}}),Il=new g({props:{code:"cGlwJTIwaW5zdGFsbCUyMGdhbG9yZS10b3JjaA==",highlighted:"pip install galore-torch",wrap:!1}}),Al=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBkYXRhc2V0cyUwQWltcG9ydCUyMHRybCUwQSUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMEF1dG9Db25maWclMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEF0cmFpbl9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCdpbWRiJyUyQyUyMHNwbGl0JTNEJ3RyYWluJyklMEElMEFhcmdzJTIwJTNEJTIwVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMi4lMkZ0ZXN0LWdhbG9yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDEwMCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbSUzRCUyMmdhbG9yZV9hZGFtdyUyMiUyQyUwQSUyMCUyMCUyMCUyMG9wdGltX3RhcmdldF9tb2R1bGVzJTNEJTVCJTIyYXR0biUyMiUyQyUyMCUyMm1scCUyMiU1RCUwQSklMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyJTBBJTBBY29uZmlnJTIwJTNEJTIwQXV0b0NvbmZpZy5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX2NvbmZpZyhjb25maWcpLnRvKDApJTBBJTBBdHJhaW5lciUyMCUzRCUyMHRybC5TRlRUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMjAlMEElMjAlMjAlMjAlMjBhcmdzJTNEYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0cmFpbl9kYXRhc2V0JTJDJTBBJTIwJTIwJTIwJTIwZGF0YXNldF90ZXh0X2ZpZWxkJTNEJ3RleHQnJTJDJTBBJTIwJTIwJTIwJTIwbWF4X3NlcV9sZW5ndGglM0Q1MTIlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> datasets
<span class="hljs-keyword">import</span> trl
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, AutoConfig, AutoTokenizer, AutoModelForCausalLM
train_dataset = datasets.load_dataset(<span class="hljs-string">&#x27;imdb&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
args = TrainingArguments(
 output_dir=<span class="hljs-string">&quot;./test-galore&quot;</span>,
 max_steps=<span class="hljs-number">100</span>,
 per_device_train_batch_size=<span class="hljs-number">2</span>,
 optim=<span class="hljs-string">&quot;galore_adamw&quot;</span>,
 optim_target_modules=[<span class="hljs-string">&quot;attn&quot;</span>, <span class="hljs-string">&quot;mlp&quot;</span>]
)
model_id = <span class="hljs-string">&quot;google/gemma-2b&quot;</span>
config = AutoConfig.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_config(config).to(<span class="hljs-number">0</span>)
trainer = trl.SFTTrainer(
 model=model,
args=args,
 train_dataset=train_dataset,
 dataset_text_field=<span class="hljs-string">&#x27;text&#x27;</span>,
 max_seq_length=<span class="hljs-number">512</span>,
)
trainer.train()`,wrap:!1}}),Zl=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBkYXRhc2V0cyUwQWltcG9ydCUyMHRybCUwQSUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMEF1dG9Db25maWclMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEF0cmFpbl9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCdpbWRiJyUyQyUyMHNwbGl0JTNEJ3RyYWluJyklMEElMEFhcmdzJTIwJTNEJTIwVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMi4lMkZ0ZXN0LWdhbG9yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDEwMCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbSUzRCUyMmdhbG9yZV9hZGFtdyUyMiUyQyUwQSUyMCUyMCUyMCUyMG9wdGltX3RhcmdldF9tb2R1bGVzJTNEJTVCJTIyYXR0biUyMiUyQyUyMCUyMm1scCUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMG9wdGltX2FyZ3MlM0QlMjJyYW5rJTNENjQlMkMlMjB1cGRhdGVfcHJval9nYXAlM0QxMDAlMkMlMjBzY2FsZSUzRDAuMTAlMjIlMkMlMEEpJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJnb29nbGUlMkZnZW1tYS0yYiUyMiUwQSUwQWNvbmZpZyUyMCUzRCUyMEF1dG9Db25maWcuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9jb25maWcoY29uZmlnKS50bygwKSUwQSUwQXRyYWluZXIlMjAlM0QlMjB0cmwuU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTIwJTBBJTIwJTIwJTIwJTIwYXJncyUzRGFyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGRhdGFzZXRfdGV4dF9maWVsZCUzRCd0ZXh0JyUyQyUwQSUyMCUyMCUyMCUyMG1heF9zZXFfbGVuZ3RoJTNENTEyJTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> datasets
<span class="hljs-keyword">import</span> trl
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, AutoConfig, AutoTokenizer, AutoModelForCausalLM
train_dataset = datasets.load_dataset(<span class="hljs-string">&#x27;imdb&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
args = TrainingArguments(
 output_dir=<span class="hljs-string">&quot;./test-galore&quot;</span>,
 max_steps=<span class="hljs-number">100</span>,
 per_device_train_batch_size=<span class="hljs-number">2</span>,
 optim=<span class="hljs-string">&quot;galore_adamw&quot;</span>,
 optim_target_modules=[<span class="hljs-string">&quot;attn&quot;</span>, <span class="hljs-string">&quot;mlp&quot;</span>],
 optim_args=<span class="hljs-string">&quot;rank=64, update_proj_gap=100, scale=0.10&quot;</span>,
)
model_id = <span class="hljs-string">&quot;google/gemma-2b&quot;</span>
config = AutoConfig.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_config(config).to(<span class="hljs-number">0</span>)
trainer = trl.SFTTrainer(
 model=model,
args=args,
 train_dataset=train_dataset,
 dataset_text_field=<span class="hljs-string">&#x27;text&#x27;</span>,
 max_seq_length=<span class="hljs-number">512</span>,
)
trainer.train()`,wrap:!1}}),Ql=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBkYXRhc2V0cyUwQWltcG9ydCUyMHRybCUwQSUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMEF1dG9Db25maWclMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEF0cmFpbl9kYXRhc2V0JTIwJTNEJTIwZGF0YXNldHMubG9hZF9kYXRhc2V0KCdpbWRiJyUyQyUyMHNwbGl0JTNEJ3RyYWluJyklMEElMEFhcmdzJTIwJTNEJTIwVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMi4lMkZ0ZXN0LWdhbG9yZSUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDEwMCUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbSUzRCUyMmdhbG9yZV9hZGFtd19sYXllcndpc2UlMjIlMkMlMEElMjAlMjAlMjAlMjBvcHRpbV90YXJnZXRfbW9kdWxlcyUzRCU1QiUyMmF0dG4lMjIlMkMlMjAlMjJtbHAlMjIlNUQlMEEpJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJnb29nbGUlMkZnZW1tYS0yYiUyMiUwQSUwQWNvbmZpZyUyMCUzRCUyMEF1dG9Db25maWcuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9jb25maWcoY29uZmlnKS50bygwKSUwQSUwQXRyYWluZXIlMjAlM0QlMjB0cmwuU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTIwJTBBJTIwJTIwJTIwJTIwYXJncyUzRGFyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGRhdGFzZXRfdGV4dF9maWVsZCUzRCd0ZXh0JyUyQyUwQSUyMCUyMCUyMCUyMG1heF9zZXFfbGVuZ3RoJTNENTEyJTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> datasets
<span class="hljs-keyword">import</span> trl
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, AutoConfig, AutoTokenizer, AutoModelForCausalLM
train_dataset = datasets.load_dataset(<span class="hljs-string">&#x27;imdb&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
args = TrainingArguments(
 output_dir=<span class="hljs-string">&quot;./test-galore&quot;</span>,
 max_steps=<span class="hljs-number">100</span>,
 per_device_train_batch_size=<span class="hljs-number">2</span>,
 optim=<span class="hljs-string">&quot;galore_adamw_layerwise&quot;</span>,
 optim_target_modules=[<span class="hljs-string">&quot;attn&quot;</span>, <span class="hljs-string">&quot;mlp&quot;</span>]
)
model_id = <span class="hljs-string">&quot;google/gemma-2b&quot;</span>
config = AutoConfig.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_config(config).to(<span class="hljs-number">0</span>)
trainer = trl.SFTTrainer(
 model=model,
args=args,
 train_dataset=train_dataset,
 dataset_text_field=<span class="hljs-string">&#x27;text&#x27;</span>,
 max_seq_length=<span class="hljs-number">512</span>,
)
trainer.train()`,wrap:!1}}),Fl=new Z({props:{title:"LOMO 옵티마이저",local:"lomo-optimizer",headingTag:"h2"}}),X=new Kl({props:{$$slots:{default:[Ca]},$$scope:{ctx:h}}}),kl=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBkYXRhc2V0cyUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQWltcG9ydCUyMHRybCUwQSUwQXRyYWluX2RhdGFzZXQlMjAlM0QlMjBkYXRhc2V0cy5sb2FkX2RhdGFzZXQoJ2ltZGInJTJDJTIwc3BsaXQlM0QndHJhaW4nKSUwQSUwQWFyZ3MlMjAlM0QlMjBUcmFpbmluZ0FyZ3VtZW50cyglMEElMjAlMjAlMjAlMjBvdXRwdXRfZGlyJTNEJTIyLiUyRnRlc3QtbG9tbyUyMiUyQyUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzRDEwMDAlMkMlMEElMjAlMjAlMjAlMjBwZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwb3B0aW0lM0QlMjJhZGFsb21vJTIyJTJDJTBBJTIwJTIwJTIwJTIwZ3JhZGllbnRfY2hlY2twb2ludGluZyUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0cmF0ZWd5JTNEJTIyc3RlcHMlMjIlMkMlMEElMjAlMjAlMjAlMjBsb2dnaW5nX3N0ZXBzJTNEMSUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QyZS02JTJDJTBBJTIwJTIwJTIwJTIwc2F2ZV9zdHJhdGVneSUzRCUyMm5vJTIyJTJDJTBBJTIwJTIwJTIwJTIwcnVuX25hbWUlM0QlMjJsb21vLWltZGIlMjIlMkMlMEEpJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJnb29nbGUlMkZnZW1tYS0yYiUyMiUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwbG93X2NwdV9tZW1fdXNhZ2UlM0RUcnVlKS50bygwKSUwQSUwQXRyYWluZXIlMjAlM0QlMjB0cmwuU0ZUVHJhaW5lciglMEElMjAlMjAlMjAlMjBtb2RlbCUzRG1vZGVsJTJDJTIwJTBBJTIwJTIwJTIwJTIwYXJncyUzRGFyZ3MlMkMlMEElMjAlMjAlMjAlMjB0cmFpbl9kYXRhc2V0JTNEdHJhaW5fZGF0YXNldCUyQyUwQSUyMCUyMCUyMCUyMGRhdGFzZXRfdGV4dF9maWVsZCUzRCd0ZXh0JyUyQyUwQSUyMCUyMCUyMCUyMG1heF9zZXFfbGVuZ3RoJTNEMTAyNCUyQyUwQSklMEElMEF0cmFpbmVyLnRyYWluKCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> datasets
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, AutoTokenizer, AutoModelForCausalLM
<span class="hljs-keyword">import</span> trl
train_dataset = datasets.load_dataset(<span class="hljs-string">&#x27;imdb&#x27;</span>, split=<span class="hljs-string">&#x27;train&#x27;</span>)
args = TrainingArguments(
 output_dir=<span class="hljs-string">&quot;./test-lomo&quot;</span>,
 max_steps=<span class="hljs-number">1000</span>,
 per_device_train_batch_size=<span class="hljs-number">4</span>,
 optim=<span class="hljs-string">&quot;adalomo&quot;</span>,
 gradient_checkpointing=<span class="hljs-literal">True</span>,
 logging_strategy=<span class="hljs-string">&quot;steps&quot;</span>,
 logging_steps=<span class="hljs-number">1</span>,
 learning_rate=<span class="hljs-number">2e-6</span>,
 save_strategy=<span class="hljs-string">&quot;no&quot;</span>,
 run_name=<span class="hljs-string">&quot;lomo-imdb&quot;</span>,
)
model_id = <span class="hljs-string">&quot;google/gemma-2b&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, low_cpu_mem_usage=<span class="hljs-literal">True</span>).to(<span class="hljs-number">0</span>)
trainer = trl.SFTTrainer(
 model=model,
args=args,
 train_dataset=train_dataset,
 dataset_text_field=<span class="hljs-string">&#x27;text&#x27;</span>,
 max_seq_length=<span class="hljs-number">1024</span>,
)
trainer.train()`,wrap:!1}}),Nl=new Z({props:{title:"Accelerate와 Trainer",local:"accelerate-and-trainer",headingTag:"h2"}}),Q=new Kl({props:{$$slots:{default:[Za]},$$scope:{ctx:h}}}),R=new Ja({props:{id:"config",options:["DistributedDataParallel","FSDP","DeepSpeed","DeepSpeed with Accelerate plugin"],$$slots:{default:[Qa]},$$scope:{ctx:h}}}),Hl=new g({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMCU1QyUwQSUyMCUyMCUyMCUyMC4lMkZleGFtcGxlcyUyRnB5dG9yY2glMkZ0ZXh0LWNsYXNzaWZpY2F0aW9uJTJGcnVuX2dsdWUucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1vZGVsX25hbWVfb3JfcGF0aCUyMGdvb2dsZS1iZXJ0JTJGYmVydC1iYXNlLWNhc2VkJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS10YXNrX25hbWUlMjAlMjRUQVNLX05BTUUlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRvX3RyYWluJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kb19ldmFsJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXhfc2VxX2xlbmd0aCUyMDEyOCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwMTYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWxlYXJuaW5nX3JhdGUlMjA1ZS01JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1udW1fdHJhaW5fZXBvY2hzJTIwMyUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRiUyNFRBU0tfTkFNRSUyRiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3ZlcndyaXRlX291dHB1dF9kaXI=",highlighted:`accelerate launch \\
 ./examples/pytorch/text-classification/run_glue.py \\
 --model_name_or_path google-bert/bert-base-cased \\
 --task_name <span class="hljs-variable">$TASK_NAME</span> \\
 --do_train \\
 --do_eval \\
 --max_seq_length 128 \\
 --per_device_train_batch_size 16 \\
 --learning_rate 5e-5 \\
 --num_train_epochs 3 \\
 --output_dir /tmp/<span class="hljs-variable">$TASK_NAME</span>/ \\
 --overwrite_output_dir`,wrap:!1}}),xl=new g({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbnVtX3Byb2Nlc3NlcyUzRDIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXVzZV9mc2RwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1taXhlZF9wcmVjaXNpb24lM0RiZjE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX2F1dG9fd3JhcF9wb2xpY3klM0RUUkFOU0ZPUk1FUl9CQVNFRF9XUkFQJTIwJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX3RyYW5zZm9ybWVyX2xheWVyX2Nsc190b193cmFwJTNEJTIyQmVydExheWVyJTIyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1mc2RwX3NoYXJkaW5nX3N0cmF0ZWd5JTNEMSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZnNkcF9zdGF0ZV9kaWN0X3R5cGUlM0RGVUxMX1NUQVRFX0RJQ1QlMjAlNUMlMEElMjAlMjAlMjAlMjAuJTJGZXhhbXBsZXMlMkZweXRvcmNoJTJGdGV4dC1jbGFzc2lmaWNhdGlvbiUyRnJ1bl9nbHVlLnB5JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBnb29nbGUtYmVydCUyRmJlcnQtYmFzZS1jYXNlZCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdGFza19uYW1lJTIwJTI0VEFTS19OQU1FJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kb190cmFpbiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZG9fZXZhbCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbWF4X3NlcV9sZW5ndGglMjAxMjglMjAlNUMlMEElMjAlMjAlMjAlMjAtLXBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMDE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sZWFybmluZ19yYXRlJTIwNWUtNSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbnVtX3RyYWluX2Vwb2NocyUyMDMlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkYlMjRUQVNLX05BTUUlMkYlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW92ZXJ3cml0ZV9vdXRwdXRfZGly",highlighted:`accelerate launch --num_processes=2 \\
 --use_fsdp \\
 --mixed_precision=bf16 \\
 --fsdp_auto_wrap_policy=TRANSFORMER_BASED_WRAP \\
 --fsdp_transformer_layer_cls_to_wrap=<span class="hljs-string">&quot;BertLayer&quot;</span> \\
 --fsdp_sharding_strategy=1 \\
 --fsdp_state_dict_type=FULL_STATE_DICT \\
 ./examples/pytorch/text-classification/run_glue.py \\
 --model_name_or_path google-bert/bert-base-cased \\
 --task_name <span class="hljs-variable">$TASK_NAME</span> \\
 --do_train \\
 --do_eval \\
 --max_seq_length 128 \\
 --per_device_train_batch_size 16 \\
 --learning_rate 5e-5 \\
 --num_train_epochs 3 \\
 --output_dir /tmp/<span class="hljs-variable">$TASK_NAME</span>/ \\
 --overwrite_output_dir`,wrap:!1}}),Dl=new ua({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/ko/trainer.md"}}),{c(){t=i("meta"),M=r(),n=i("p"),d=r(),J(m.$$.fragment),u=r(),_=i("p"),_.innerHTML=I,T=r(),J(b.$$.fragment),F=r(),C=i("p"),C.innerHTML=Pl,E=r(),J(W.$$.fragment),ls=r(),k=i("p"),k.innerHTML=fe,ss=r(),J(N.$$.fragment),es=r(),G=i("p"),G.innerHTML=de,as=r(),v=i("ol"),v.innerHTML=ue,ts=r(),S=i("p"),S.innerHTML=he,ns=r(),Y=i("p"),Y.innerHTML=_e,rs=r(),J(H.$$.fragment),ps=r(),z=i("p"),z.innerHTML=be,Ms=r(),x=i("p"),x.innerHTML=Ie,is=r(),J(L.$$.fragment),cs=r(),J(D.$$.fragment),os=r(),q=i("p"),q.innerHTML=ge,ms=r(),J(P.$$.fragment),Ts=r(),O=i("p"),O.innerHTML=Ae,Js=r(),K=i("ul"),K.innerHTML=Ce,js=r(),ll=i("p"),ll.innerHTML=Ze,ws=r(),J(sl.$$.fragment),ys=r(),el=i("p"),el.innerHTML=Ve,Us=r(),al=i("ul"),al.innerHTML=$e,fs=r(),tl=i("p"),tl.innerHTML=Be,ds=r(),J(nl.$$.fragment),us=r(),J(rl.$$.fragment),hs=r(),pl=i("p"),pl.innerHTML=Xe,_s=r(),Ml=i("p"),Ml.textContent=Qe,bs=r(),J(il.$$.fragment),Is=r(),cl=i("p"),cl.innerHTML=Re,gs=r(),J(ol.$$.fragment),As=r(),J(ml.$$.fragment),Cs=r(),J(V.$$.fragment),Zs=r(),Tl=i("p"),Tl.innerHTML=Fe,Vs=r(),Jl=i("p"),Jl.innerHTML=Ee,$s=r(),J($.$$.fragment),Bs=r(),jl=i("p"),jl.textContent=We,Xs=r(),J(wl.$$.fragment),Qs=r(),yl=i("p"),yl.innerHTML=ke,Rs=r(),J(B.$$.fragment),Fs=r(),J(Ul.$$.fragment),Es=r(),fl=i("p"),fl.innerHTML=Ne,Ws=r(),J(dl.$$.fragment),ks=r(),ul=i("p"),ul.textContent=Ge,Ns=r(),J(hl.$$.fragment),Gs=r(),_l=i("p"),_l.textContent=ve,vs=r(),bl=i("p"),bl.textContent=Se,Ss=r(),J(Il.$$.fragment),Ys=r(),gl=i("p"),gl.innerHTML=Ye,Hs=r(),J(Al.$$.fragment),zs=r(),Cl=i("p"),Cl.innerHTML=He,xs=r(),J(Zl.$$.fragment),Ls=r(),Vl=i("p"),Vl.innerHTML=ze,Ds=r(),$l=i("p"),$l.textContent=xe,qs=r(),Bl=i("p"),Bl.textContent=Le,Ps=r(),Xl=i("p"),Xl.innerHTML=De,Os=r(),J(Ql.$$.fragment),Ks=r(),Rl=i("p"),Rl.innerHTML=qe,le=r(),J(Fl.$$.fragment),se=r(),El=i("p"),El.innerHTML=Pe,ee=r(),J(X.$$.fragment),ae=r(),Wl=i("p"),Wl.innerHTML=Oe,te=r(),J(kl.$$.fragment),ne=r(),J(Nl.$$.fragment),re=r(),Gl=i("p"),Gl.innerHTML=Ke,pe=r(),J(Q.$$.fragment),Me=r(),vl=i("p"),vl.innerHTML=la,ie=r(),J(R.$$.fragment),ce=r(),Sl=i("p"),Sl.innerHTML=sa,oe=r(),Yl=i("p"),Yl.innerHTML=ea,me=r(),J(Hl.$$.fragment),Te=r(),zl=i("p"),zl.innerHTML=aa,Je=r(),J(xl.$$.fragment),je=r(),Ll=i("p"),Ll.innerHTML=ta,we=r(),J(Dl.$$.fragment),ye=r(),Ol=i("p"),this.h()},l(l){const s=fa("svelte-u9bgzb",document.head);t=c(s,"META",{name:!0,content:!0}),s.forEach(e),M=p(l),n=c(l,"P",{}),ma(n).forEach(e),d=p(l),j(m.$$.fragment,l),u=p(l),_=c(l,"P",{"data-svelte-h":!0}),o(_)!=="svelte-z2bqhq"&&(_.innerHTML=I),T=p(l),j(b.$$.fragment,l),F=p(l),C=c(l,"P",{"data-svelte-h":!0}),o(C)!=="svelte-1cp0146"&&(C.innerHTML=Pl),E=p(l),j(W.$$.fragment,l),ls=p(l),k=c(l,"P",{"data-svelte-h":!0}),o(k)!=="svelte-qdq8ju"&&(k.innerHTML=fe),ss=p(l),j(N.$$.fragment,l),es=p(l),G=c(l,"P",{"data-svelte-h":!0}),o(G)!=="svelte-1hnjlfe"&&(G.innerHTML=de),as=p(l),v=c(l,"OL",{"data-svelte-h":!0}),o(v)!=="svelte-1paahjl"&&(v.innerHTML=ue),ts=p(l),S=c(l,"P",{"data-svelte-h":!0}),o(S)!=="svelte-13opt4p"&&(S.innerHTML=he),ns=p(l),Y=c(l,"P",{"data-svelte-h":!0}),o(Y)!=="svelte-w28uq4"&&(Y.innerHTML=_e),rs=p(l),j(H.$$.fragment,l),ps=p(l),z=c(l,"P",{"data-svelte-h":!0}),o(z)!=="svelte-1wzpi4r"&&(z.innerHTML=be),Ms=p(l),x=c(l,"P",{"data-svelte-h":!0}),o(x)!=="svelte-1kos7wh"&&(x.innerHTML=Ie),is=p(l),j(L.$$.fragment,l),cs=p(l),j(D.$$.fragment,l),os=p(l),q=c(l,"P",{"data-svelte-h":!0}),o(q)!=="svelte-1eveoyv"&&(q.innerHTML=ge),ms=p(l),j(P.$$.fragment,l),Ts=p(l),O=c(l,"P",{"data-svelte-h":!0}),o(O)!=="svelte-16kdh7t"&&(O.innerHTML=Ae),Js=p(l),K=c(l,"UL",{"data-svelte-h":!0}),o(K)!=="svelte-1biqkkg"&&(K.innerHTML=Ce),js=p(l),ll=c(l,"P",{"data-svelte-h":!0}),o(ll)!=="svelte-1lsqood"&&(ll.innerHTML=Ze),ws=p(l),j(sl.$$.fragment,l),ys=p(l),el=c(l,"P",{"data-svelte-h":!0}),o(el)!=="svelte-h5n0c3"&&(el.innerHTML=Ve),Us=p(l),al=c(l,"UL",{"data-svelte-h":!0}),o(al)!=="svelte-v8vsyj"&&(al.innerHTML=$e),fs=p(l),tl=c(l,"P",{"data-svelte-h":!0}),o(tl)!=="svelte-1f6dirw"&&(tl.innerHTML=Be),ds=p(l),j(nl.$$.fragment,l),us=p(l),j(rl.$$.fragment,l),hs=p(l),pl=c(l,"P",{"data-svelte-h":!0}),o(pl)!=="svelte-12l51bo"&&(pl.innerHTML=Xe),_s=p(l),Ml=c(l,"P",{"data-svelte-h":!0}),o(Ml)!=="svelte-1sr1d63"&&(Ml.textContent=Qe),bs=p(l),j(il.$$.fragment,l),Is=p(l),cl=c(l,"P",{"data-svelte-h":!0}),o(cl)!=="svelte-6czbxl"&&(cl.innerHTML=Re),gs=p(l),j(ol.$$.fragment,l),As=p(l),j(ml.$$.fragment,l),Cs=p(l),j(V.$$.fragment,l),Zs=p(l),Tl=c(l,"P",{"data-svelte-h":!0}),o(Tl)!=="svelte-1ai0l0q"&&(Tl.innerHTML=Fe),Vs=p(l),Jl=c(l,"P",{"data-svelte-h":!0}),o(Jl)!=="svelte-1xaqk62"&&(Jl.innerHTML=Ee),$s=p(l),j($.$$.fragment,l),Bs=p(l),jl=c(l,"P",{"data-svelte-h":!0}),o(jl)!=="svelte-1dt04b5"&&(jl.textContent=We),Xs=p(l),j(wl.$$.fragment,l),Qs=p(l),yl=c(l,"P",{"data-svelte-h":!0}),o(yl)!=="svelte-1dk6uay"&&(yl.innerHTML=ke),Rs=p(l),j(B.$$.fragment,l),Fs=p(l),j(Ul.$$.fragment,l),Es=p(l),fl=c(l,"P",{"data-svelte-h":!0}),o(fl)!=="svelte-1c4tyr9"&&(fl.innerHTML=Ne),Ws=p(l),j(dl.$$.fragment,l),ks=p(l),ul=c(l,"P",{"data-svelte-h":!0}),o(ul)!=="svelte-1ft34jp"&&(ul.textContent=Ge),Ns=p(l),j(hl.$$.fragment,l),Gs=p(l),_l=c(l,"P",{"data-svelte-h":!0}),o(_l)!=="svelte-zig1fw"&&(_l.textContent=ve),vs=p(l),bl=c(l,"P",{"data-svelte-h":!0}),o(bl)!=="svelte-15aiu37"&&(bl.textContent=Se),Ss=p(l),j(Il.$$.fragment,l),Ys=p(l),gl=c(l,"P",{"data-svelte-h":!0}),o(gl)!=="svelte-13s90m9"&&(gl.innerHTML=Ye),Hs=p(l),j(Al.$$.fragment,l),zs=p(l),Cl=c(l,"P",{"data-svelte-h":!0}),o(Cl)!=="svelte-q0mtk0"&&(Cl.innerHTML=He),xs=p(l),j(Zl.$$.fragment,l),Ls=p(l),Vl=c(l,"P",{"data-svelte-h":!0}),o(Vl)!=="svelte-9wx27h"&&(Vl.innerHTML=ze),Ds=p(l),$l=c(l,"P",{"data-svelte-h":!0}),o($l)!=="svelte-3m2n0g"&&($l.textContent=xe),qs=p(l),Bl=c(l,"P",{"data-svelte-h":!0}),o(Bl)!=="svelte-o488v2"&&(Bl.textContent=Le),Ps=p(l),Xl=c(l,"P",{"data-svelte-h":!0}),o(Xl)!=="svelte-6sty29"&&(Xl.innerHTML=De),Os=p(l),j(Ql.$$.fragment,l),Ks=p(l),Rl=c(l,"P",{"data-svelte-h":!0}),o(Rl)!=="svelte-1h6bbfe"&&(Rl.innerHTML=qe),le=p(l),j(Fl.$$.fragment,l),se=p(l),El=c(l,"P",{"data-svelte-h":!0}),o(El)!=="svelte-19j518g"&&(El.innerHTML=Pe),ee=p(l),j(X.$$.fragment,l),ae=p(l),Wl=c(l,"P",{"data-svelte-h":!0}),o(Wl)!=="svelte-16ffc3o"&&(Wl.innerHTML=Oe),te=p(l),j(kl.$$.fragment,l),ne=p(l),j(Nl.$$.fragment,l),re=p(l),Gl=c(l,"P",{"data-svelte-h":!0}),o(Gl)!=="svelte-10c072n"&&(Gl.innerHTML=Ke),pe=p(l),j(Q.$$.fragment,l),Me=p(l),vl=c(l,"P",{"data-svelte-h":!0}),o(vl)!=="svelte-17ew3hs"&&(vl.innerHTML=la),ie=p(l),j(R.$$.fragment,l),ce=p(l),Sl=c(l,"P",{"data-svelte-h":!0}),o(Sl)!=="svelte-1l2gru4"&&(Sl.innerHTML=sa),oe=p(l),Yl=c(l,"P",{"data-svelte-h":!0}),o(Yl)!=="svelte-uvyyka"&&(Yl.innerHTML=ea),me=p(l),j(Hl.$$.fragment,l),Te=p(l),zl=c(l,"P",{"data-svelte-h":!0}),o(zl)!=="svelte-1nohodh"&&(zl.innerHTML=aa),Je=p(l),j(xl.$$.fragment,l),je=p(l),Ll=c(l,"P",{"data-svelte-h":!0}),o(Ll)!=="svelte-1xu5fm6"&&(Ll.innerHTML=ta),we=p(l),j(Dl.$$.fragment,l),ye=p(l),Ol=c(l,"P",{}),ma(Ol).forEach(e),this.h()},h(){Ta(t,"name","hf:doc:metadata"),Ta(t,"content",Fa)},m(l,s){da(document.head,t),a(l,M,s),a(l,n,s),a(l,d,s),w(m,l,s),a(l,u,s),a(l,_,s),a(l,T,s),w(b,l,s),a(l,F,s),a(l,C,s),a(l,E,s),w(W,l,s),a(l,ls,s),a(l,k,s),a(l,ss,s),w(N,l,s),a(l,es,s),a(l,G,s),a(l,as,s),a(l,v,s),a(l,ts,s),a(l,S,s),a(l,ns,s),a(l,Y,s),a(l,rs,s),w(H,l,s),a(l,ps,s),a(l,z,s),a(l,Ms,s),a(l,x,s),a(l,is,s),w(L,l,s),a(l,cs,s),w(D,l,s),a(l,os,s),a(l,q,s),a(l,ms,s),w(P,l,s),a(l,Ts,s),a(l,O,s),a(l,Js,s),a(l,K,s),a(l,js,s),a(l,ll,s),a(l,ws,s),w(sl,l,s),a(l,ys,s),a(l,el,s),a(l,Us,s),a(l,al,s),a(l,fs,s),a(l,tl,s),a(l,ds,s),w(nl,l,s),a(l,us,s),w(rl,l,s),a(l,hs,s),a(l,pl,s),a(l,_s,s),a(l,Ml,s),a(l,bs,s),w(il,l,s),a(l,Is,s),a(l,cl,s),a(l,gs,s),w(ol,l,s),a(l,As,s),w(ml,l,s),a(l,Cs,s),w(V,l,s),a(l,Zs,s),a(l,Tl,s),a(l,Vs,s),a(l,Jl,s),a(l,$s,s),w($,l,s),a(l,Bs,s),a(l,jl,s),a(l,Xs,s),w(wl,l,s),a(l,Qs,s),a(l,yl,s),a(l,Rs,s),w(B,l,s),a(l,Fs,s),w(Ul,l,s),a(l,Es,s),a(l,fl,s),a(l,Ws,s),w(dl,l,s),a(l,ks,s),a(l,ul,s),a(l,Ns,s),w(hl,l,s),a(l,Gs,s),a(l,_l,s),a(l,vs,s),a(l,bl,s),a(l,Ss,s),w(Il,l,s),a(l,Ys,s),a(l,gl,s),a(l,Hs,s),w(Al,l,s),a(l,zs,s),a(l,Cl,s),a(l,xs,s),w(Zl,l,s),a(l,Ls,s),a(l,Vl,s),a(l,Ds,s),a(l,$l,s),a(l,qs,s),a(l,Bl,s),a(l,Ps,s),a(l,Xl,s),a(l,Os,s),w(Ql,l,s),a(l,Ks,s),a(l,Rl,s),a(l,le,s),w(Fl,l,s),a(l,se,s),a(l,El,s),a(l,ee,s),w(X,l,s),a(l,ae,s),a(l,Wl,s),a(l,te,s),w(kl,l,s),a(l,ne,s),w(Nl,l,s),a(l,re,s),a(l,Gl,s),a(l,pe,s),w(Q,l,s),a(l,Me,s),a(l,vl,s),a(l,ie,s),w(R,l,s),a(l,ce,s),a(l,Sl,s),a(l,oe,s),a(l,Yl,s),a(l,me,s),w(Hl,l,s),a(l,Te,s),a(l,zl,s),a(l,Je,s),w(xl,l,s),a(l,je,s),a(l,Ll,s),a(l,we,s),w(Dl,l,s),a(l,ye,s),a(l,Ol,s),Ue=!0},p(l,[s]){const na={};s&2&&(na.$$scope={dirty:s,ctx:l}),b.$set(na);const ra={};s&2&&(ra.$$scope={dirty:s,ctx:l}),V.$set(ra);const pa={};s&2&&(pa.$$scope={dirty:s,ctx:l}),$.$set(pa);const Ma={};s&2&&(Ma.$$scope={dirty:s,ctx:l}),B.$set(Ma);const ia={};s&2&&(ia.$$scope={dirty:s,ctx:l}),X.$set(ia);const ca={};s&2&&(ca.$$scope={dirty:s,ctx:l}),Q.$set(ca);const oa={};s&2&&(oa.$$scope={dirty:s,ctx:l}),R.$set(oa)},i(l){Ue||(y(m.$$.fragment,l),y(b.$$.fragment,l),y(W.$$.fragment,l),y(N.$$.fragment,l),y(H.$$.fragment,l),y(L.$$.fragment,l),y(D.$$.fragment,l),y(P.$$.fragment,l),y(sl.$$.fragment,l),y(nl.$$.fragment,l),y(rl.$$.fragment,l),y(il.$$.fragment,l),y(ol.$$.fragment,l),y(ml.$$.fragment,l),y(V.$$.fragment,l),y($.$$.fragment,l),y(wl.$$.fragment,l),y(B.$$.fragment,l),y(Ul.$$.fragment,l),y(dl.$$.fragment,l),y(hl.$$.fragment,l),y(Il.$$.fragment,l),y(Al.$$.fragment,l),y(Zl.$$.fragment,l),y(Ql.$$.fragment,l),y(Fl.$$.fragment,l),y(X.$$.fragment,l),y(kl.$$.fragment,l),y(Nl.$$.fragment,l),y(Q.$$.fragment,l),y(R.$$.fragment,l),y(Hl.$$.fragment,l),y(xl.$$.fragment,l),y(Dl.$$.fragment,l),Ue=!0)},o(l){U(m.$$.fragment,l),U(b.$$.fragment,l),U(W.$$.fragment,l),U(N.$$.fragment,l),U(H.$$.fragment,l),U(L.$$.fragment,l),U(D.$$.fragment,l),U(P.$$.fragment,l),U(sl.$$.fragment,l),U(nl.$$.fragment,l),U(rl.$$.fragment,l),U(il.$$.fragment,l),U(ol.$$.fragment,l),U(ml.$$.fragment,l),U(V.$$.fragment,l),U($.$$.fragment,l),U(wl.$$.fragment,l),U(B.$$.fragment,l),U(Ul.$$.fragment,l),U(dl.$$.fragment,l),U(hl.$$.fragment,l),U(Il.$$.fragment,l),U(Al.$$.fragment,l),U(Zl.$$.fragment,l),U(Ql.$$.fragment,l),U(Fl.$$.fragment,l),U(X.$$.fragment,l),U(kl.$$.fragment,l),U(Nl.$$.fragment,l),U(Q.$$.fragment,l),U(R.$$.fragment,l),U(Hl.$$.fragment,l),U(xl.$$.fragment,l),U(Dl.$$.fragment,l),Ue=!1},d(l){l&&(e(M),e(n),e(d),e(u),e(_),e(T),e(F),e(C),e(E),e(ls),e(k),e(ss),e(es),e(G),e(as),e(v),e(ts),e(S),e(ns),e(Y),e(rs),e(ps),e(z),e(Ms),e(x),e(is),e(cs),e(os),e(q),e(ms),e(Ts),e(O),e(Js),e(K),e(js),e(ll),e(ws),e(ys),e(el),e(Us),e(al),e(fs),e(tl),e(ds),e(us),e(hs),e(pl),e(_s),e(Ml),e(bs),e(Is),e(cl),e(gs),e(As),e(Cs),e(Zs),e(Tl),e(Vs),e(Jl),e($s),e(Bs),e(jl),e(Xs),e(Qs),e(yl),e(Rs),e(Fs),e(Es),e(fl),e(Ws),e(ks),e(ul),e(Ns),e(Gs),e(_l),e(vs),e(bl),e(Ss),e(Ys),e(gl),e(Hs),e(zs),e(Cl),e(xs),e(Ls),e(Vl),e(Ds),e($l),e(qs),e(Bl),e(Ps),e(Xl),e(Os),e(Ks),e(Rl),e(le),e(se),e(El),e(ee),e(ae),e(Wl),e(te),e(ne),e(re),e(Gl),e(pe),e(Me),e(vl),e(ie),e(ce),e(Sl),e(oe),e(Yl),e(me),e(Te),e(zl),e(Je),e(je),e(Ll),e(we),e(ye),e(Ol)),e(t),f(m,l),f(b,l),f(W,l),f(N,l),f(H,l),f(L,l),f(D,l),f(P,l),f(sl,l),f(nl,l),f(rl,l),f(il,l),f(ol,l),f(ml,l),f(V,l),f($,l),f(wl,l),f(B,l),f(Ul,l),f(dl,l),f(hl,l),f(Il,l),f(Al,l),f(Zl,l),f(Ql,l),f(Fl,l),f(X,l),f(kl,l),f(Nl,l),f(Q,l),f(R,l),f(Hl,l),f(xl,l),f(Dl,l)}}}const Fa='{"title":"Trainer","local":"trainer","sections":[{"title":"기본 사용법","local":"basic-usage","sections":[{"title":"체크포인트","local":"checkpoints","sections":[],"depth":3}],"depth":2},{"title":"Trainer 맞춤 설정","local":"customize-the-trainer","sections":[{"title":"콜백","local":"callbacks","sections":[],"depth":3}],"depth":2},{"title":"로깅","local":"logging","sections":[],"depth":2},{"title":"NEFTune","local":"neftune","sections":[],"depth":2},{"title":"GaLore","local":"galore","sections":[],"depth":2},{"title":"LOMO 옵티마이저","local":"lomo-optimizer","sections":[],"depth":2},{"title":"Accelerate와 Trainer","local":"accelerate-and-trainer","sections":[],"depth":2}],"depth":1}';function Ea(h){return wa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ya extends ya{constructor(t){super(),Ua(this,t,Ea,Ra,ja,{})}}export{Ya as component};

Xet Storage Details

Size:
79.3 kB
·
Xet hash:
72e46dc424f005e7ba3a385bc9391fd52c06e8c23c9a1db95d811d9676f9d769

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.