Buckets:

hf-doc-build
/

doc-dev

hf-doc-build/doc-dev / transformers /pr_37082 /ko /_app /immutable /nodes /151.3aebcd55.js

rtrm's picture

3 months ago

76.7 kB

	import{s as ra,o as ia,n as cs}from"../chunks/scheduler.bdbef820.js";import{S as ca,i as oa,g as b,s as i,r as j,A as ma,h as d,f as t,c,j as na,u as h,x as J,k as ea,y as Ma,a as n,v as u,d as y,t as f,w as g,m as ja,n as ha}from"../chunks/index.33f81d56.js";import{T as Wl}from"../chunks/Tip.34194030.js";import{Y as pa}from"../chunks/Youtube.0e329b00.js";import{C as k}from"../chunks/CodeBlock.362b34a4.js";import{D as ua}from"../chunks/DocNotebookDropdown.d5db5928.js";import{F as Bl,M as Ls}from"../chunks/Markdown.03194dea.js";import{H as Ss,E as ya}from"../chunks/EditOnGithub.a9246e21.js";function fa(_){let a,o='이 작업과 호환되는 모든 아키텍처와 체크포인트를 보려면 <a href="https://huggingface.co/tasks/token-classification" rel="nofollow">작업 페이지</a>를 확인하는 것이 좋습니다.';return{c(){a=b("p"),a.innerHTML=o},l(l){a=d(l,"P",{"data-svelte-h":!0}),J(a)!=="svelte-1xdu3h"&&(a.innerHTML=o)},m(l,m){n(l,a,m)},p:cs,d(l){l&&t(a)}}}function ga(_){let a,o;return a=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

	<span class="hljs-meta">>>> </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`,wrap:!1}}),{c(){j(a.$$.fragment)},l(l){h(a.$$.fragment,l)},m(l,m){u(a,l,m),o=!0},p:cs,i(l){o\|\|(y(a.$$.fragment,l),o=!0)},o(l){f(a.$$.fragment,l),o=!1},d(l){g(a,l)}}}function ba(_){let a,o;return a=new Ls({props:{$$slots:{default:[ga]},$$scope:{ctx:_}}}),{c(){j(a.$$.fragment)},l(l){h(a.$$.fragment,l)},m(l,m){u(a,l,m),o=!0},p(l,m){const w={};m&2&&(w.$$scope={dirty:m,ctx:l}),a.$set(w)},i(l){o\|\|(y(a.$$.fragment,l),o=!0)},o(l){f(a.$$.fragment,l),o=!1},d(l){g(a,l)}}}function da(_){let a,o;return a=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

	<span class="hljs-meta">>>> </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">"tf"</span>)`,wrap:!1}}),{c(){j(a.$$.fragment)},l(l){h(a.$$.fragment,l)},m(l,m){u(a,l,m),o=!0},p:cs,i(l){o\|\|(y(a.$$.fragment,l),o=!0)},o(l){f(a.$$.fragment,l),o=!1},d(l){g(a,l)}}}function wa(_){let a,o;return a=new Ls({props:{$$slots:{default:[da]},$$scope:{ctx:_}}}),{c(){j(a.$$.fragment)},l(l){h(a.$$.fragment,l)},m(l,m){u(a,l,m),o=!0},p(l,m){const w={};m&2&&(w.$$scope={dirty:m,ctx:l}),a.$set(w)},i(l){o\|\|(y(a.$$.fragment,l),o=!0)},o(l){f(a.$$.fragment,l),o=!1},d(l){g(a,l)}}}function Ja(_){let a,o='<a href="/docs/transformers/pr_37082/ko/main_classes/trainer#transformers.Trainer">Trainer</a>를 사용하여 모델을 파인 튜닝하는 방법에 익숙하지 않은 경우, <a href="../training#train-with-pytorch-trainer">여기</a>에서 기본 튜토리얼을 확인하세요!';return{c(){a=b("p"),a.innerHTML=o},l(l){a=d(l,"P",{"data-svelte-h":!0}),J(a)!=="svelte-im1w58"&&(a.innerHTML=o)},m(l,m){n(l,a,m)},p:cs,d(l){l&&t(a)}}}function Ta(_){let a,o,l,m='이제 모델을 훈련시킬 준비가 되었습니다! <a href="/docs/transformers/pr_37082/ko/model_doc/auto#transformers.AutoModelForSequenceClassification">AutoModelForSequenceClassification</a>로 DistilBERT를 가져오고 예상되는 레이블 수와 레이블 매핑을 지정하세요:',w,C,A,v,I="이제 세 단계만 거치면 끝입니다:",G,x,V='<li><a href="/docs/transformers/pr_37082/ko/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>에서 하이퍼파라미터를 정의하세요. <code>output_dir</code>는 모델을 저장할 위치를 지정하는 유일한 매개변수입니다. 이 모델을 허브에 업로드하기 위해 <code>push_to_hub=True</code>를 설정합니다(모델을 업로드하기 위해 Hugging Face에 로그인해야합니다.) 각 에폭이 끝날 때마다, <a href="/docs/transformers/pr_37082/ko/main_classes/trainer#transformers.Trainer">Trainer</a>는 seqeval 점수를 평가하고 훈련 체크포인트를 저장합니다.</li> <li><a href="/docs/transformers/pr_37082/ko/main_classes/trainer#transformers.Trainer">Trainer</a>에 훈련 인수와 모델, 데이터 세트, 토크나이저, 데이터 콜레이터 및 <code>compute_metrics</code> 함수를 전달하세요.</li> <li><a href="/docs/transformers/pr_37082/ko/main_classes/trainer#transformers.Trainer.train">train()</a>를 호출하여 모델을 파인 튜닝하세요.</li>',W,$,B,r,U='훈련이 완료되면, <a href="/docs/transformers/pr_37082/ko/main_classes/trainer#transformers.Trainer.push_to_hub">push_to_hub()</a> 메소드를 사용하여 모델을 허브에 공유할 수 있습니다.',E,X,R;return a=new Wl({props:{$$slots:{default:[Ja]},$$scope:{ctx:_}}}),C=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMFRyYWluZXIlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmRpc3RpbGJlcnQlMkZkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0QxMyUyQyUyMGlkMmxhYmVsJTNEaWQybGFiZWwlMkMlMjBsYWJlbDJpZCUzRGxhYmVsMmlkJTBBKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

	<span class="hljs-meta">>>> </span>model = AutoModelForTokenClassification.from_pretrained(
	<span class="hljs-meta">... </span> <span class="hljs-string">"distilbert/distilbert-base-uncased"</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
	<span class="hljs-meta">... </span>)`,wrap:!1}}),$=new k({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QyJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5JTNEMC4wMSUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxvYWRfYmVzdF9tb2RlbF9hdF9lbmQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0b2tlbml6ZWRfd251dCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3dudXQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwcHJvY2Vzc2luZ19jbGFzcyUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMGRhdGFfY29sbGF0b3IlM0RkYXRhX2NvbGxhdG9yJTJDJTBBJTIwJTIwJTIwJTIwY29tcHV0ZV9tZXRyaWNzJTNEY29tcHV0ZV9tZXRyaWNzJTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-meta">>>> </span>training_args = TrainingArguments(
	<span class="hljs-meta">... </span> output_dir=<span class="hljs-string">"my_awesome_wnut_model"</span>,
	<span class="hljs-meta">... </span> learning_rate=<span class="hljs-number">2e-5</span>,
	<span class="hljs-meta">... </span> per_device_train_batch_size=<span class="hljs-number">16</span>,
	<span class="hljs-meta">... </span> per_device_eval_batch_size=<span class="hljs-number">16</span>,
	<span class="hljs-meta">... </span> num_train_epochs=<span class="hljs-number">2</span>,
	<span class="hljs-meta">... </span> weight_decay=<span class="hljs-number">0.01</span>,
	<span class="hljs-meta">... </span> eval_strategy=<span class="hljs-string">"epoch"</span>,
	<span class="hljs-meta">... </span> save_strategy=<span class="hljs-string">"epoch"</span>,
	<span class="hljs-meta">... </span> load_best_model_at_end=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> push_to_hub=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>trainer = Trainer(
	<span class="hljs-meta">... </span> model=model,
	<span class="hljs-meta">... </span> args=training_args,
	<span class="hljs-meta">... </span> train_dataset=tokenized_wnut[<span class="hljs-string">"train"</span>],
	<span class="hljs-meta">... </span> eval_dataset=tokenized_wnut[<span class="hljs-string">"test"</span>],
	<span class="hljs-meta">... </span> processing_class=tokenizer,
	<span class="hljs-meta">... </span> data_collator=data_collator,
	<span class="hljs-meta">... </span> compute_metrics=compute_metrics,
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>trainer.train()`,wrap:!1}}),X=new k({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">>>> </span>trainer.push_to_hub()',wrap:!1}}),{c(){j(a.$$.fragment),o=i(),l=b("p"),l.innerHTML=m,w=i(),j(C.$$.fragment),A=i(),v=b("p"),v.textContent=I,G=i(),x=b("ol"),x.innerHTML=V,W=i(),j($.$$.fragment),B=i(),r=b("p"),r.innerHTML=U,E=i(),j(X.$$.fragment)},l(M){h(a.$$.fragment,M),o=c(M),l=d(M,"P",{"data-svelte-h":!0}),J(l)!=="svelte-q50myw"&&(l.innerHTML=m),w=c(M),h(C.$$.fragment,M),A=c(M),v=d(M,"P",{"data-svelte-h":!0}),J(v)!=="svelte-14zzcxs"&&(v.textContent=I),G=c(M),x=d(M,"OL",{"data-svelte-h":!0}),J(x)!=="svelte-15i40mw"&&(x.innerHTML=V),W=c(M),h($.$$.fragment,M),B=c(M),r=d(M,"P",{"data-svelte-h":!0}),J(r)!=="svelte-1o91fdi"&&(r.innerHTML=U),E=c(M),h(X.$$.fragment,M)},m(M,Z){u(a,M,Z),n(M,o,Z),n(M,l,Z),n(M,w,Z),u(C,M,Z),n(M,A,Z),n(M,v,Z),n(M,G,Z),n(M,x,Z),n(M,W,Z),u($,M,Z),n(M,B,Z),n(M,r,Z),n(M,E,Z),u(X,M,Z),R=!0},p(M,Z){const H={};Z&2&&(H.$$scope={dirty:Z,ctx:M}),a.$set(H)},i(M){R\|\|(y(a.$$.fragment,M),y(C.$$.fragment,M),y($.$$.fragment,M),y(X.$$.fragment,M),R=!0)},o(M){f(a.$$.fragment,M),f(C.$$.fragment,M),f($.$$.fragment,M),f(X.$$.fragment,M),R=!1},d(M){M&&(t(o),t(l),t(w),t(A),t(v),t(G),t(x),t(W),t(B),t(r),t(E)),g(a,M),g(C,M),g($,M),g(X,M)}}}function Ua(_){let a,o;return a=new Ls({props:{$$slots:{default:[Ta]},$$scope:{ctx:_}}}),{c(){j(a.$$.fragment)},l(l){h(a.$$.fragment,l)},m(l,m){u(a,l,m),o=!0},p(l,m){const w={};m&2&&(w.$$scope={dirty:m,ctx:l}),a.$set(w)},i(l){o\|\|(y(a.$$.fragment,l),o=!0)},o(l){f(a.$$.fragment,l),o=!1},d(l){g(a,l)}}}function $a(_){let a,o='Keras를 사용하여 모델을 파인 튜닝하는 방법에 익숙하지 않은 경우, <a href="../training#train-a-tensorflow-model-with-keras">여기</a>의 기본 튜토리얼을 확인하세요!';return{c(){a=b("p"),a.innerHTML=o},l(l){a=d(l,"P",{"data-svelte-h":!0}),J(a)!=="svelte-nkj0lu"&&(a.innerHTML=o)},m(l,m){n(l,a,m)},p:cs,d(l){l&&t(a)}}}function xa(_){let a,o,l,m,w,C='그런 다음 <a href="/docs/transformers/pr_37082/ko/model_doc/auto#transformers.TFAutoModelForSequenceClassification">TFAutoModelForSequenceClassification</a>을 사용하여 DistilBERT를 가져오고, 예상되는 레이블 수와 레이블 매핑을 지정합니다:',A,v,I,G,x='<a href="/docs/transformers/pr_37082/ko/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset">prepare_tf_dataset()</a>을 사용하여 데이터 세트를 <code>tf.data.Dataset</code> 형식으로 변환합니다:',V,W,$,B,r='<a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a>를 사용하여 훈련할 모델을 구성합니다:',U,E,X,R,M='훈련을 시작하기 전에 설정해야할 마지막 두 가지는 예측에서 seqeval 점수를 계산하고, 모델을 허브에 업로드할 방법을 제공하는 것입니다. 모두 <a href="../main_classes/keras_callbacks">Keras callbacks</a>를 사용하여 수행됩니다.',Z,H,os='<a href="/docs/transformers/pr_37082/ko/main_classes/keras_callbacks#transformers.KerasMetricCallback">KerasMetricCallback</a>에 <code>compute_metrics</code> 함수를 전달하세요:',F,z,Q,ls,ms='<a href="/docs/transformers/pr_37082/ko/main_classes/keras_callbacks#transformers.PushToHubCallback">PushToHubCallback</a>에서 모델과 토크나이저를 업로드할 위치를 지정합니다:',N,Y,q,S,as="그런 다음 콜백을 함께 묶습니다:",Ms,L,D,P,ts='드디어, 모델 훈련을 시작할 준비가 되었습니다! <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a>에 훈련 데이터 세트, 검증 데이터 세트, 에폭의 수 및 콜백을 전달하여 파인 튜닝합니다:',js,K,O,ss,ns="훈련이 완료되면, 모델이 자동으로 허브에 업로드되어 누구나 사용할 수 있습니다!",hs;return a=new Wl({props:{$$slots:{default:[$a]},$$scope:{ctx:_}}}),l=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMChsZW4odG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fdHJhaW5fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX3N0ZXBzJTNEbnVtX3RyYWluX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5X3JhdGUlM0QwLjAxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEEp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

	<span class="hljs-meta">>>> </span>batch_size = <span class="hljs-number">16</span>
	<span class="hljs-meta">>>> </span>num_train_epochs = <span class="hljs-number">3</span>
	<span class="hljs-meta">>>> </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">"train"</span>]) // batch_size) * num_train_epochs
	<span class="hljs-meta">>>> </span>optimizer, lr_schedule = create_optimizer(
	<span class="hljs-meta">... </span> init_lr=<span class="hljs-number">2e-5</span>,
	<span class="hljs-meta">... </span> num_train_steps=num_train_steps,
	<span class="hljs-meta">... </span> weight_decay_rate=<span class="hljs-number">0.01</span>,
	<span class="hljs-meta">... </span> num_warmup_steps=<span class="hljs-number">0</span>,
	<span class="hljs-meta">... </span>)`,wrap:!1}}),v=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJkaXN0aWxiZXJ0JTJGZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQlMjIlMkMlMjBudW1fbGFiZWxzJTNEMTMlMkMlMjBpZDJsYWJlbCUzRGlkMmxhYmVsJTJDJTIwbGFiZWwyaWQlM0RsYWJlbDJpZCUwQSk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

	<span class="hljs-meta">>>> </span>model = TFAutoModelForTokenClassification.from_pretrained(
	<span class="hljs-meta">... </span> <span class="hljs-string">"distilbert/distilbert-base-uncased"</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
	<span class="hljs-meta">... </span>)`,wrap:!1}}),W=new k({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydHJhaW4lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKSUwQSUwQXRmX3ZhbGlkYXRpb25fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydmFsaWRhdGlvbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKQ==",highlighted:`<span class="hljs-meta">>>> </span>tf_train_set = model.prepare_tf_dataset(
	<span class="hljs-meta">... </span> tokenized_wnut[<span class="hljs-string">"train"</span>],
	<span class="hljs-meta">... </span> shuffle=<span class="hljs-literal">True</span>,
	<span class="hljs-meta">... </span> batch_size=<span class="hljs-number">16</span>,
	<span class="hljs-meta">... </span> collate_fn=data_collator,
	<span class="hljs-meta">... </span>)

	<span class="hljs-meta">>>> </span>tf_validation_set = model.prepare_tf_dataset(
	<span class="hljs-meta">... </span> tokenized_wnut[<span class="hljs-string">"validation"</span>],
	<span class="hljs-meta">... </span> shuffle=<span class="hljs-literal">False</span>,
	<span class="hljs-meta">... </span> batch_size=<span class="hljs-number">16</span>,
	<span class="hljs-meta">... </span> collate_fn=data_collator,
	<span class="hljs-meta">... </span>)`,wrap:!1}}),E=new k({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplcik=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

	<span class="hljs-meta">>>> </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`,wrap:!1}}),z=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBLZXJhc01ldHJpY0NhbGxiYWNrJTBBJTBBbWV0cmljX2NhbGxiYWNrJTIwJTNEJTIwS2VyYXNNZXRyaWNDYWxsYmFjayhtZXRyaWNfZm4lM0Rjb21wdXRlX21ldHJpY3MlMkMlMjBldmFsX2RhdGFzZXQlM0R0Zl92YWxpZGF0aW9uX3NldCk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> KerasMetricCallback

	<span class="hljs-meta">>>> </span>metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)`,wrap:!1}}),Y=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQXB1c2hfdG9faHViX2NhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2soJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

	<span class="hljs-meta">>>> </span>push_to_hub_callback = PushToHubCallback(
	<span class="hljs-meta">... </span> output_dir=<span class="hljs-string">"my_awesome_wnut_model"</span>,
	<span class="hljs-meta">... </span> tokenizer=tokenizer,
	<span class="hljs-meta">... </span>)`,wrap:!1}}),L=new k({props:{code:"Y2FsbGJhY2tzJTIwJTNEJTIwJTVCbWV0cmljX2NhbGxiYWNrJTJDJTIwcHVzaF90b19odWJfY2FsbGJhY2slNUQ=",highlighted:'<span class="hljs-meta">>>> </span>callbacks = [metric_callback, push_to_hub_callback]',wrap:!1}}),K=new k({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMlMkMlMjBjYWxsYmFja3MlM0RjYWxsYmFja3Mp",highlighted:'<span class="hljs-meta">>>> </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>, callbacks=callbacks)',wrap:!1}}),{c(){j(a.$$.fragment),o=ja(`
	TensorFlow에서 모델을 파인 튜닝하려면, 먼저 옵티마이저 함수와 학습률 스케쥴, 그리고 일부 훈련 하이퍼파라미터를 설정해야 합니다:

	`),j(l.$$.fragment),m=i(),w=b("p"),w.innerHTML=C,A=i(),j(v.$$.fragment),I=i(),G=b("p"),G.innerHTML=x,V=i(),j(W.$$.fragment),$=i(),B=b("p"),B.innerHTML=r,U=i(),j(E.$$.fragment),X=i(),R=b("p"),R.innerHTML=M,Z=i(),H=b("p"),H.innerHTML=os,F=i(),j(z.$$.fragment),Q=i(),ls=b("p"),ls.innerHTML=ms,N=i(),j(Y.$$.fragment),q=i(),S=b("p"),S.textContent=as,Ms=i(),j(L.$$.fragment),D=i(),P=b("p"),P.innerHTML=ts,js=i(),j(K.$$.fragment),O=i(),ss=b("p"),ss.textContent=ns},l(p){h(a.$$.fragment,p),o=ha(p,`
	TensorFlow에서 모델을 파인 튜닝하려면, 먼저 옵티마이저 함수와 학습률 스케쥴, 그리고 일부 훈련 하이퍼파라미터를 설정해야 합니다:

	`),h(l.$$.fragment,p),m=c(p),w=d(p,"P",{"data-svelte-h":!0}),J(w)!=="svelte-1gvbo6y"&&(w.innerHTML=C),A=c(p),h(v.$$.fragment,p),I=c(p),G=d(p,"P",{"data-svelte-h":!0}),J(G)!=="svelte-zhcnd2"&&(G.innerHTML=x),V=c(p),h(W.$$.fragment,p),$=c(p),B=d(p,"P",{"data-svelte-h":!0}),J(B)!=="svelte-qrlpiv"&&(B.innerHTML=r),U=c(p),h(E.$$.fragment,p),X=c(p),R=d(p,"P",{"data-svelte-h":!0}),J(R)!=="svelte-10vlwlw"&&(R.innerHTML=M),Z=c(p),H=d(p,"P",{"data-svelte-h":!0}),J(H)!=="svelte-1etku35"&&(H.innerHTML=os),F=c(p),h(z.$$.fragment,p),Q=c(p),ls=d(p,"P",{"data-svelte-h":!0}),J(ls)!=="svelte-1ihtg9o"&&(ls.innerHTML=ms),N=c(p),h(Y.$$.fragment,p),q=c(p),S=d(p,"P",{"data-svelte-h":!0}),J(S)!=="svelte-90s2we"&&(S.textContent=as),Ms=c(p),h(L.$$.fragment,p),D=c(p),P=d(p,"P",{"data-svelte-h":!0}),J(P)!=="svelte-1kbr19n"&&(P.innerHTML=ts),js=c(p),h(K.$$.fragment,p),O=c(p),ss=d(p,"P",{"data-svelte-h":!0}),J(ss)!=="svelte-w14up1"&&(ss.textContent=ns)},m(p,T){u(a,p,T),n(p,o,T),u(l,p,T),n(p,m,T),n(p,w,T),n(p,A,T),u(v,p,T),n(p,I,T),n(p,G,T),n(p,V,T),u(W,p,T),n(p,$,T),n(p,B,T),n(p,U,T),u(E,p,T),n(p,X,T),n(p,R,T),n(p,Z,T),n(p,H,T),n(p,F,T),u(z,p,T),n(p,Q,T),n(p,ls,T),n(p,N,T),u(Y,p,T),n(p,q,T),n(p,S,T),n(p,Ms,T),u(L,p,T),n(p,D,T),n(p,P,T),n(p,js,T),u(K,p,T),n(p,O,T),n(p,ss,T),hs=!0},p(p,T){const us={};T&2&&(us.$$scope={dirty:T,ctx:p}),a.$set(us)},i(p){hs\|\|(y(a.$$.fragment,p),y(l.$$.fragment,p),y(v.$$.fragment,p),y(W.$$.fragment,p),y(E.$$.fragment,p),y(z.$$.fragment,p),y(Y.$$.fragment,p),y(L.$$.fragment,p),y(K.$$.fragment,p),hs=!0)},o(p){f(a.$$.fragment,p),f(l.$$.fragment,p),f(v.$$.fragment,p),f(W.$$.fragment,p),f(E.$$.fragment,p),f(z.$$.fragment,p),f(Y.$$.fragment,p),f(L.$$.fragment,p),f(K.$$.fragment,p),hs=!1},d(p){p&&(t(o),t(m),t(w),t(A),t(I),t(G),t(V),t($),t(B),t(U),t(X),t(R),t(Z),t(H),t(F),t(Q),t(ls),t(N),t(q),t(S),t(Ms),t(D),t(P),t(js),t(O),t(ss)),g(a,p),g(l,p),g(v,p),g(W,p),g(E,p),g(z,p),g(Y,p),g(L,p),g(K,p)}}}function ka(_){let a,o;return a=new Ls({props:{$$slots:{default:[xa]},$$scope:{ctx:_}}}),{c(){j(a.$$.fragment)},l(l){h(a.$$.fragment,l)},m(l,m){u(a,l,m),o=!0},p(l,m){const w={};m&2&&(w.$$scope={dirty:m,ctx:l}),a.$set(w)},i(l){o\|\|(y(a.$$.fragment,l),o=!0)},o(l){f(a.$$.fragment,l),o=!1},d(l){g(a,l)}}}function _a(_){let a,o=`토큰 분류를 위한 모델을 파인 튜닝하는 자세한 예제는 다음
	<a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb" rel="nofollow">PyTorch notebook</a>
	또는 <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb" rel="nofollow">TensorFlow notebook</a>를 참조하세요.`;return{c(){a=b("p"),a.innerHTML=o},l(l){a=d(l,"P",{"data-svelte-h":!0}),J(a)!=="svelte-139pooy"&&(a.innerHTML=o)},m(l,m){n(l,a,m)},p:cs,d(l){l&&t(a)}}}function Ca(_){let a,o="텍스트를 토큰화하고 PyTorch 텐서를 반환합니다:",l,m,w,C,A="입력을 모델에 전달하고 <code>logits</code>을 반환합니다:",v,I,G,x,V="가장 높은 확률을 가진 클래스를 모델의 <code>id2label</code> 매핑을 사용하여 텍스트 레이블로 변환합니다:",W,$,B;return m=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

	<span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"stevhliu/my_awesome_wnut_model"</span>)
	<span class="hljs-meta">>>> </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">"pt"</span>)`,wrap:!1}}),I=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMnN0ZXZobGl1JTJGbXlfYXdlc29tZV93bnV0X21vZGVsJTIyKSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification

	<span class="hljs-meta">>>> </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">"stevhliu/my_awesome_wnut_model"</span>)
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad():
	<span class="hljs-meta">... </span> logits = model(**inputs).logits`,wrap:!1}}),$=new k({props:{code:"cHJlZGljdGlvbnMlMjAlM0QlMjB0b3JjaC5hcmdtYXgobG9naXRzJTJDJTIwZGltJTNEMiklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0Lml0ZW0oKSU1RCUyMGZvciUyMHQlMjBpbiUyMHByZWRpY3Rpb25zJTVCMCU1RCU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">>>> </span>predictions = torch.argmax(logits, dim=<span class="hljs-number">2</span>)
	<span class="hljs-meta">>>> </span>predicted_token_class = [model.config.id2label[t.item()] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predictions[<span class="hljs-number">0</span>]]
	<span class="hljs-meta">>>> </span>predicted_token_class
	[<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'I-location'</span>,
	<span class="hljs-string">'B-group'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>]`,wrap:!1}}),{c(){a=b("p"),a.textContent=o,l=i(),j(m.$$.fragment),w=i(),C=b("p"),C.innerHTML=A,v=i(),j(I.$$.fragment),G=i(),x=b("p"),x.innerHTML=V,W=i(),j($.$$.fragment)},l(r){a=d(r,"P",{"data-svelte-h":!0}),J(a)!=="svelte-ctuaol"&&(a.textContent=o),l=c(r),h(m.$$.fragment,r),w=c(r),C=d(r,"P",{"data-svelte-h":!0}),J(C)!=="svelte-1hjuppo"&&(C.innerHTML=A),v=c(r),h(I.$$.fragment,r),G=c(r),x=d(r,"P",{"data-svelte-h":!0}),J(x)!=="svelte-1jbp04u"&&(x.innerHTML=V),W=c(r),h($.$$.fragment,r)},m(r,U){n(r,a,U),n(r,l,U),u(m,r,U),n(r,w,U),n(r,C,U),n(r,v,U),u(I,r,U),n(r,G,U),n(r,x,U),n(r,W,U),u($,r,U),B=!0},p:cs,i(r){B\|\|(y(m.$$.fragment,r),y(I.$$.fragment,r),y($.$$.fragment,r),B=!0)},o(r){f(m.$$.fragment,r),f(I.$$.fragment,r),f($.$$.fragment,r),B=!1},d(r){r&&(t(a),t(l),t(w),t(C),t(v),t(G),t(x),t(W)),g(m,r),g(I,r),g($,r)}}}function Ia(_){let a,o;return a=new Ls({props:{$$slots:{default:[Ca]},$$scope:{ctx:_}}}),{c(){j(a.$$.fragment)},l(l){h(a.$$.fragment,l)},m(l,m){u(a,l,m),o=!0},p(l,m){const w={};m&2&&(w.$$scope={dirty:m,ctx:l}),a.$set(w)},i(l){o\|\|(y(a.$$.fragment,l),o=!0)},o(l){f(a.$$.fragment,l),o=!1},d(l){g(a,l)}}}function Za(_){let a,o="텍스트를 토큰화하고 TensorFlow 텐서를 반환합니다:",l,m,w,C,A="입력값을 모델에 전달하고 <code>logits</code>을 반환합니다:",v,I,G,x,V="가장 높은 확률을 가진 클래스를 모델의 <code>id2label</code> 매핑을 사용하여 텍스트 레이블로 변환합니다:",W,$,B;return m=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

	<span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"stevhliu/my_awesome_wnut_model"</span>)
	<span class="hljs-meta">>>> </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">"tf"</span>)`,wrap:!1}}),I=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

	<span class="hljs-meta">>>> </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">"stevhliu/my_awesome_wnut_model"</span>)
	<span class="hljs-meta">>>> </span>logits = model(**inputs).logits`,wrap:!1}}),$=new k({props:{code:"cHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyUyMCUzRCUyMHRmLm1hdGguYXJnbWF4KGxvZ2l0cyUyQyUyMGF4aXMlM0QtMSklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0JTVEJTIwZm9yJTIwdCUyMGluJTIwcHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyU1QjAlNUQubnVtcHkoKS50b2xpc3QoKSU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">>>> </span>predicted_token_class_ids = tf.math.argmax(logits, axis=-<span class="hljs-number">1</span>)
	<span class="hljs-meta">>>> </span>predicted_token_class = [model.config.id2label[t] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predicted_token_class_ids[<span class="hljs-number">0</span>].numpy().tolist()]
	<span class="hljs-meta">>>> </span>predicted_token_class
	[<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'I-location'</span>,
	<span class="hljs-string">'B-group'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'O'</span>,
	<span class="hljs-string">'O'</span>]`,wrap:!1}}),{c(){a=b("p"),a.textContent=o,l=i(),j(m.$$.fragment),w=i(),C=b("p"),C.innerHTML=A,v=i(),j(I.$$.fragment),G=i(),x=b("p"),x.innerHTML=V,W=i(),j($.$$.fragment)},l(r){a=d(r,"P",{"data-svelte-h":!0}),J(a)!=="svelte-1m12ipr"&&(a.textContent=o),l=c(r),h(m.$$.fragment,r),w=c(r),C=d(r,"P",{"data-svelte-h":!0}),J(C)!=="svelte-tcnp5y"&&(C.innerHTML=A),v=c(r),h(I.$$.fragment,r),G=c(r),x=d(r,"P",{"data-svelte-h":!0}),J(x)!=="svelte-1jbp04u"&&(x.innerHTML=V),W=c(r),h($.$$.fragment,r)},m(r,U){n(r,a,U),n(r,l,U),u(m,r,U),n(r,w,U),n(r,C,U),n(r,v,U),u(I,r,U),n(r,G,U),n(r,x,U),n(r,W,U),u($,r,U),B=!0},p:cs,i(r){B\|\|(y(m.$$.fragment,r),y(I.$$.fragment,r),y($.$$.fragment,r),B=!0)},o(r){f(m.$$.fragment,r),f(I.$$.fragment,r),f($.$$.fragment,r),B=!1},d(r){r&&(t(a),t(l),t(w),t(C),t(v),t(G),t(x),t(W)),g(m,r),g(I,r),g($,r)}}}function va(_){let a,o;return a=new Ls({props:{$$slots:{default:[Za]},$$scope:{ctx:_}}}),{c(){j(a.$$.fragment)},l(l){h(a.$$.fragment,l)},m(l,m){u(a,l,m),o=!0},p(l,m){const w={};m&2&&(w.$$scope={dirty:m,ctx:l}),a.$set(w)},i(l){o\|\|(y(a.$$.fragment,l),o=!0)},o(l){f(a.$$.fragment,l),o=!1},d(l){g(a,l)}}}function Ga(_){let a,o,l,m,w,C,A,v,I,G,x,V="토큰 분류는 문장의 개별 토큰에 레이블을 할당합니다. 가장 일반적인 토큰 분류 작업 중 하나는 개체명 인식(Named Entity Recognition, NER)입니다. 개체명 인식은 문장에서 사람, 위치 또는 조직과 같은 각 개체의 레이블을 찾으려고 시도합니다.",W,$,B="이 가이드에서 학습할 내용은:",r,U,E='<li><a href="https://huggingface.co/datasets/wnut_17" rel="nofollow">WNUT 17</a> 데이터 세트에서 <a href="https://huggingface.co/distilbert/distilbert-base-uncased" rel="nofollow">DistilBERT</a>를 파인 튜닝하여 새로운 개체를 탐지합니다.</li> <li>추론을 위해 파인 튜닝 모델을 사용합니다.</li>',X,R,M,Z,H="시작하기 전에, 필요한 모든 라이브러리가 설치되어 있는지 확인하세요:",os,F,z,Q,ls="Hugging Face 계정에 로그인하여 모델을 업로드하고 커뮤니티에 공유하는 것을 권장합니다. 메시지가 표시되면, 토큰을 입력하여 로그인하세요:",ms,N,Y,q,S,as,Ms="먼저 🤗 Datasets 라이브러리에서 WNUT 17 데이터 세트를 가져옵니다:",L,D,P,ts,js="다음 예제를 살펴보세요:",K,O,ss,ns,hs="<code>ner_tags</code>의 각 숫자는 개체를 나타냅니다. 숫자를 레이블 이름으로 변환하여 개체가 무엇인지 확인합니다:",p,T,us,ys,Al="각 <code>ner_tag</code>의 앞에 붙은 문자는 개체의 토큰 위치를 나타냅니다:",Ps,fs,Rl="<li><code>B-</code>는 개체의 시작을 나타냅니다.</li> <li><code>I-</code>는 토큰이 동일한 개체 내부에 포함되어 있음을 나타냅니다(예를 들어 <code>State</code> 토큰은 <code>Empire State Building</code>와 같은 개체의 일부입니다).</li> <li><code>0</code>는 토큰이 어떤 개체에도 해당하지 않음을 나타냅니다.</li>",Ks,gs,Os,bs,sl,ds,Xl="다음으로 <code>tokens</code> 필드를 전처리하기 위해 DistilBERT 토크나이저를 가져옵니다:",ll,ws,al,Js,Vl="위의 예제 <code>tokens</code> 필드를 보면 입력이 이미 토큰화된 것처럼 보입니다. 그러나 실제로 입력은 아직 토큰화되지 않았으므로 단어를 하위 단어로 토큰화하기 위해 <code>is_split_into_words=True</code>를 설정해야 합니다. 예제로 확인합니다:",tl,Ts,nl,Us,El="그러나 이로 인해 <code>[CLS]</code>과 <code>[SEP]</code>라는 특수 토큰이 추가되고, 하위 단어 토큰화로 인해 입력과 레이블 간에 불일치가 발생합니다. 하나의 레이블에 해당하는 단일 단어는 이제 두 개의 하위 단어로 분할될 수 있습니다. 토큰과 레이블을 다음과 같이 재정렬해야 합니다:",el,$s,Hl='<li><a href="https://huggingface.co/docs/transformers/main_classes/tokenizer#transformers.BatchEncoding.word_ids" rel="nofollow"><code>word_ids</code></a> 메소드로 모든 토큰을 해당 단어에 매핑합니다.</li> <li>특수 토큰 <code>[CLS]</code>와 <code>[SEP]</code>에 <code>-100</code> 레이블을 할당하여, PyTorch 손실 함수가 해당 토큰을 무시하도록 합니다.</li> <li>주어진 단어의 첫 번째 토큰에만 레이블을 지정합니다. 같은 단어의 다른 하위 토큰에 <code>-100</code>을 할당합니다.</li>',pl,xs,Fl="다음은 토큰과 레이블을 재정렬하고 DistilBERT의 최대 입력 길이보다 길지 않도록 시퀀스를 잘라내는 함수를 만드는 방법입니다:",rl,ks,il,_s,zl="전체 데이터 세트에 전처리 함수를 적용하려면, 🤗 Datasets <code>map</code> 함수를 사용하세요. <code>batched=True</code>로 설정하여 데이터 세트의 여러 요소를 한 번에 처리하면 <code>map</code> 함수의 속도를 높일 수 있습니다:",cl,Cs,ol,Is,Ql='이제 <a href="/docs/transformers/pr_37082/ko/main_classes/data_collator#transformers.DataCollatorWithPadding">DataCollatorWithPadding</a>를 사용하여 예제 배치를 만들어봅시다. 데이터 세트 전체를 최대 길이로 패딩하는 대신, <em>동적 패딩</em>을 사용하여 배치에서 가장 긴 길이에 맞게 문장을 패딩하는 것이 효율적입니다.',ml,es,Ml,Zs,jl,vs,Nl='훈련 중 모델의 성능을 평가하기 위해 평가 지표를 포함하는 것이 유용합니다. 🤗 <a href="https://huggingface.co/docs/evaluate/index" rel="nofollow">Evaluate</a> 라이브러리를 사용하여 빠르게 평가 방법을 가져올 수 있습니다. 이 작업에서는 <a href="https://huggingface.co/spaces/evaluate-metric/seqeval" rel="nofollow">seqeval</a> 평가 지표를 가져옵니다. (평가 지표를 가져오고 계산하는 방법에 대해서는 🤗 Evaluate <a href="https://huggingface.co/docs/evaluate/a_quick_tour" rel="nofollow">빠른 둘러보기</a>를 참조하세요). Seqeval은 실제로 정밀도, 재현률, F1 및 정확도와 같은 여러 점수를 산출합니다.',hl,Gs,ul,Ws,Yl="먼저 NER 레이블을 가져온 다음, <code>compute</code>에 실제 예측과 실제 레이블을 전달하여 점수를 계산하는 함수를 만듭니다:",yl,Bs,fl,As,ql="이제 <code>compute_metrics</code> 함수를 사용할 준비가 되었으며, 훈련을 설정하면 이 함수로 되돌아올 것입니다.",gl,Rs,bl,Xs,Sl="모델을 훈련하기 전에, <code>id2label</code>와 <code>label2id</code>를 사용하여 예상되는 id와 레이블의 맵을 생성하세요:",dl,Vs,wl,ps,Jl,rs,Tl,Es,Ul,Hs,Ll="좋아요, 이제 모델을 파인 튜닝했으니 추론에 사용할 수 있습니다!",$l,Fs,Dl="추론을 수행하고자 하는 텍스트를 가져와봅시다:",xl,zs,kl,Qs,Pl="파인 튜닝된 모델로 추론을 시도하는 가장 간단한 방법은 <code>pipeline()</code>를 사용하는 것입니다. 모델로 NER의 <code>pipeline</code>을 인스턴스화하고, 텍스트를 전달해보세요:",_l,Ns,Cl,Ys,Kl="원한다면, <code>pipeline</code>의 결과를 수동으로 복제할 수도 있습니다:",Il,is,Zl,qs,vl,Ds,Gl;return w=new Ss({props:{title:"토큰 분류",local:"token-classification",headingTag:"h1"}}),A=new ua({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/token_classification.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/token_classification.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/token_classification.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/token_classification.ipynb"}]}}),I=new pa({props:{id:"wVHdVlPScxA"}}),R=new Wl({props:{$$slots:{default:[fa]},$$scope:{ctx:_}}}),F=new k({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwZXZhbHVhdGUlMjBzZXFldmFs",highlighted:"pip install transformers datasets evaluate seqeval",wrap:!1}}),N=new k({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

	<span class="hljs-meta">>>> </span>notebook_login()`,wrap:!1}}),q=new Ss({props:{title:"WNUT 17 데이터 세트 가져오기",local:"load-wnut-17-dataset",headingTag:"h2"}}),D=new k({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBd251dCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ3bnV0XzE3JTIyKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

	<span class="hljs-meta">>>> </span>wnut = load_dataset(<span class="hljs-string">"wnut_17"</span>)`,wrap:!1}}),O=new k({props:{code:"d251dCU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">>>> </span>wnut[<span class="hljs-string">"train"</span>][<span class="hljs-number">0</span>]
	{<span class="hljs-string">'id'</span>: <span class="hljs-string">'0'</span>,
	<span class="hljs-string">'ner_tags'</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
	<span class="hljs-string">'tokens'</span>: [<span class="hljs-string">'@paulwalk'</span>, <span class="hljs-string">'It'</span>, <span class="hljs-string">"'s"</span>, <span class="hljs-string">'the'</span>, <span class="hljs-string">'view'</span>, <span class="hljs-string">'from'</span>, <span class="hljs-string">'where'</span>, <span class="hljs-string">'I'</span>, <span class="hljs-string">"'m"</span>, <span class="hljs-string">'living'</span>, <span class="hljs-string">'for'</span>, <span class="hljs-string">'two'</span>, <span class="hljs-string">'weeks'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'Empire'</span>, <span class="hljs-string">'State'</span>, <span class="hljs-string">'Building'</span>, <span class="hljs-string">'='</span>, <span class="hljs-string">'ESB'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'Pretty'</span>, <span class="hljs-string">'bad'</span>, <span class="hljs-string">'storm'</span>, <span class="hljs-string">'here'</span>, <span class="hljs-string">'last'</span>, <span class="hljs-string">'evening'</span>, <span class="hljs-string">'.'</span>]
	}`,wrap:!1}}),T=new k({props:{code:"bGFiZWxfbGlzdCUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RC5mZWF0dXJlcyU1QmYlMjJuZXJfdGFncyUyMiU1RC5mZWF0dXJlLm5hbWVzJTBBbGFiZWxfbGlzdA==",highlighted:`<span class="hljs-meta">>>> </span>label_list = wnut[<span class="hljs-string">"train"</span>].features[<span class="hljs-string">f"ner_tags"</span>].feature.names
	<span class="hljs-meta">>>> </span>label_list
	[
	<span class="hljs-string">"O"</span>,
	<span class="hljs-string">"B-corporation"</span>,
	<span class="hljs-string">"I-corporation"</span>,
	<span class="hljs-string">"B-creative-work"</span>,
	<span class="hljs-string">"I-creative-work"</span>,
	<span class="hljs-string">"B-group"</span>,
	<span class="hljs-string">"I-group"</span>,
	<span class="hljs-string">"B-location"</span>,
	<span class="hljs-string">"I-location"</span>,
	<span class="hljs-string">"B-person"</span>,
	<span class="hljs-string">"I-person"</span>,
	<span class="hljs-string">"B-product"</span>,
	<span class="hljs-string">"I-product"</span>,
	]`,wrap:!1}}),gs=new Ss({props:{title:"전처리",local:"preprocess",headingTag:"h2"}}),bs=new pa({props:{id:"iY2AZYdZAr0"}}),ws=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0JTJGZGlzdGlsYmVydC1iYXNlLXVuY2FzZWQlMjIp",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

	<span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"distilbert/distilbert-base-uncased"</span>)`,wrap:!1}}),Ts=new k({props:{code:"ZXhhbXBsZSUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQlMEF0b2tlbml6ZWRfaW5wdXQlMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSU1QiUyMnRva2VucyUyMiU1RCUyQyUyMGlzX3NwbGl0X2ludG9fd29yZHMlM0RUcnVlKSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci5jb252ZXJ0X2lkc190b190b2tlbnModG9rZW5pemVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQXRva2Vucw==",highlighted:`<span class="hljs-meta">>>> </span>example = wnut[<span class="hljs-string">"train"</span>][<span class="hljs-number">0</span>]
	<span class="hljs-meta">>>> </span>tokenized_input = tokenizer(example[<span class="hljs-string">"tokens"</span>], is_split_into_words=<span class="hljs-literal">True</span>)
	<span class="hljs-meta">>>> </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">"input_ids"</span>])
	<span class="hljs-meta">>>> </span>tokens
	[<span class="hljs-string">'[CLS]'</span>, <span class="hljs-string">'@'</span>, <span class="hljs-string">'paul'</span>, <span class="hljs-string">'##walk'</span>, <span class="hljs-string">'it'</span>, <span class="hljs-string">"'"</span>, <span class="hljs-string">'s'</span>, <span class="hljs-string">'the'</span>, <span class="hljs-string">'view'</span>, <span class="hljs-string">'from'</span>, <span class="hljs-string">'where'</span>, <span class="hljs-string">'i'</span>, <span class="hljs-string">"'"</span>, <span class="hljs-string">'m'</span>, <span class="hljs-string">'living'</span>, <span class="hljs-string">'for'</span>, <span class="hljs-string">'two'</span>, <span class="hljs-string">'weeks'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'empire'</span>, <span class="hljs-string">'state'</span>, <span class="hljs-string">'building'</span>, <span class="hljs-string">'='</span>, <span class="hljs-string">'es'</span>, <span class="hljs-string">'##b'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'pretty'</span>, <span class="hljs-string">'bad'</span>, <span class="hljs-string">'storm'</span>, <span class="hljs-string">'here'</span>, <span class="hljs-string">'last'</span>, <span class="hljs-string">'evening'</span>, <span class="hljs-string">'.'</span>, <span class="hljs-string">'[SEP]'</span>]`,wrap:!1}}),ks=new k({props:{code:"ZGVmJTIwdG9rZW5pemVfYW5kX2FsaWduX2xhYmVscyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydG9rZW5zJTIyJTVEJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEElMEElMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBpJTJDJTIwbGFiZWwlMjBpbiUyMGVudW1lcmF0ZShleGFtcGxlcyU1QmYlMjJuZXJfdGFncyUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMHRva2VuaXplZF9pbnB1dHMud29yZF9pZHMoYmF0Y2hfaW5kZXglM0RpKSUyMCUyMCUyMyUyME1hcCUyMHRva2VucyUyMHRvJTIwdGhlaXIlMjByZXNwZWN0aXZlJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyME5vbmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkeCUyMGluJTIwd29yZF9pZHMlM0ElMjAlMjAlMjMlMjBTZXQlMjB0aGUlMjBzcGVjaWFsJTIwdG9rZW5zJTIwdG8lMjAtMTAwLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZHglMjBpcyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKC0xMDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxpZiUyMHdvcmRfaWR4JTIwISUzRCUyMHByZXZpb3VzX3dvcmRfaWR4JTNBJTIwJTIwJTIzJTIwT25seSUyMGxhYmVsJTIwdGhlJTIwZmlyc3QlMjB0b2tlbiUyMG9mJTIwYSUyMGdpdmVuJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKGxhYmVsJTVCd29yZF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVsX2lkcy5hcHBlbmQoLTEwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyMHdvcmRfaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzLmFwcGVuZChsYWJlbF9pZHMpJTBBJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2lucHV0cyU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMGxhYmVscyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplZF9pbnB1dHM=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
	<span class="hljs-meta">... </span> tokenized_inputs = tokenizer(examples[<span class="hljs-string">"tokens"</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

	<span class="hljs-meta">... </span> labels = []
	<span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f"ner_tags"</span>]):
	<span class="hljs-meta">... </span> word_ids = tokenized_inputs.word_ids(batch_index=i) <span class="hljs-comment"># Map tokens to their respective word.</span>
	<span class="hljs-meta">... </span> previous_word_idx = <span class="hljs-literal">None</span>
	<span class="hljs-meta">... </span> label_ids = []
	<span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids: <span class="hljs-comment"># Set the special tokens to -100.</span>
	<span class="hljs-meta">... </span> <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
	<span class="hljs-meta">... </span> label_ids.append(-<span class="hljs-number">100</span>)
	<span class="hljs-meta">... </span> <span class="hljs-keyword">elif</span> word_idx != previous_word_idx: <span class="hljs-comment"># Only label the first token of a given word.</span>
	<span class="hljs-meta">... </span> label_ids.append(label[word_idx])
	<span class="hljs-meta">... </span> <span class="hljs-keyword">else</span>:
	<span class="hljs-meta">... </span> label_ids.append(-<span class="hljs-number">100</span>)
	<span class="hljs-meta">... </span> previous_word_idx = word_idx
	<span class="hljs-meta">... </span> labels.append(label_ids)

	<span class="hljs-meta">... </span> tokenized_inputs[<span class="hljs-string">"labels"</span>] = labels
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> tokenized_inputs`,wrap:!1}}),Cs=new k({props:{code:"dG9rZW5pemVkX3dudXQlMjAlM0QlMjB3bnV0Lm1hcCh0b2tlbml6ZV9hbmRfYWxpZ25fbGFiZWxzJTJDJTIwYmF0Y2hlZCUzRFRydWUp",highlighted:'<span class="hljs-meta">>>> </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)',wrap:!1}}),es=new Bl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[wa],pytorch:[ba]},$$scope:{ctx:_}}}),Zs=new Ss({props:{title:"평가",local:"evaluation",headingTag:"h2"}}),Gs=new k({props:{code:"aW1wb3J0JTIwZXZhbHVhdGUlMEElMEFzZXFldmFsJTIwJTNEJTIwZXZhbHVhdGUubG9hZCglMjJzZXFldmFsJTIyKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> evaluate

	<span class="hljs-meta">>>> </span>seqeval = evaluate.load(<span class="hljs-string">"seqeval"</span>)`,wrap:!1}}),Bs=new k({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBbGFiZWxzJTIwJTNEJTIwJTVCbGFiZWxfbGlzdCU1QmklNUQlMjBmb3IlMjBpJTIwaW4lMjBleGFtcGxlJTVCZiUyMm5lcl90YWdzJTIyJTVEJTVEJTBBJTBBJTBBZGVmJTIwY29tcHV0ZV9tZXRyaWNzKHApJTNBJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMkMlMjBsYWJlbHMlMjAlM0QlMjBwJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMjAlM0QlMjBucC5hcmdtYXgocHJlZGljdGlvbnMlMkMlMjBheGlzJTNEMiklMEElMEElMjAlMjAlMjAlMjB0cnVlX3ByZWRpY3Rpb25zJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCbGFiZWxfbGlzdCU1QnAlNUQlMjBmb3IlMjAocCUyQyUyMGwpJTIwaW4lMjB6aXAocHJlZGljdGlvbiUyQyUyMGxhYmVsKSUyMGlmJTIwbCUyMCElM0QlMjAtMTAwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwcHJlZGljdGlvbiUyQyUyMGxhYmVsJTIwaW4lMjB6aXAocHJlZGljdGlvbnMlMkMlMjBsYWJlbHMpJTBBJTIwJTIwJTIwJTIwJTVEJTBBJTIwJTIwJTIwJTIwdHJ1ZV9sYWJlbHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUJsYWJlbF9saXN0JTVCbCU1RCUyMGZvciUyMChwJTJDJTIwbCklMjBpbiUyMHppcChwcmVkaWN0aW9uJTJDJTIwbGFiZWwpJTIwaWYlMjBsJTIwISUzRCUyMC0xMDAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBwcmVkaWN0aW9uJTJDJTIwbGFiZWwlMjBpbiUyMHppcChwcmVkaWN0aW9ucyUyQyUyMGxhYmVscyklMEElMjAlMjAlMjAlMjAlNUQlMEElMEElMjAlMjAlMjAlMjByZXN1bHRzJTIwJTNEJTIwc2VxZXZhbC5jb21wdXRlKHByZWRpY3Rpb25zJTNEdHJ1ZV9wcmVkaWN0aW9ucyUyQyUyMHJlZmVyZW5jZXMlM0R0cnVlX2xhYmVscyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwcmVjaXNpb24lMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9wcmVjaXNpb24lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZWNhbGwlMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9yZWNhbGwlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmMSUyMiUzQSUyMHJlc3VsdHMlNUIlMjJvdmVyYWxsX2YxJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWNjdXJhY3klMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9hY2N1cmFjeSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RA==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

	<span class="hljs-meta">>>> </span>labels = [label_list[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> example[<span class="hljs-string">f"ner_tags"</span>]]


	<span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">p</span>):
	<span class="hljs-meta">... </span> predictions, labels = p
	<span class="hljs-meta">... </span> predictions = np.argmax(predictions, axis=<span class="hljs-number">2</span>)

	<span class="hljs-meta">... </span> true_predictions = [
	<span class="hljs-meta">... </span> [label_list[p] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
	<span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
	<span class="hljs-meta">... </span> ]
	<span class="hljs-meta">... </span> true_labels = [
	<span class="hljs-meta">... </span> [label_list[l] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
	<span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
	<span class="hljs-meta">... </span> ]

	<span class="hljs-meta">... </span> results = seqeval.compute(predictions=true_predictions, references=true_labels)
	<span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> {
	<span class="hljs-meta">... </span> <span class="hljs-string">"precision"</span>: results[<span class="hljs-string">"overall_precision"</span>],
	<span class="hljs-meta">... </span> <span class="hljs-string">"recall"</span>: results[<span class="hljs-string">"overall_recall"</span>],
	<span class="hljs-meta">... </span> <span class="hljs-string">"f1"</span>: results[<span class="hljs-string">"overall_f1"</span>],
	<span class="hljs-meta">... </span> <span class="hljs-string">"accuracy"</span>: results[<span class="hljs-string">"overall_accuracy"</span>],
	<span class="hljs-meta">... </span> }`,wrap:!1}}),Rs=new Ss({props:{title:"훈련",local:"train",headingTag:"h2"}}),Vs=new k({props:{code:"aWQybGFiZWwlMjAlM0QlMjAlN0IlMEElMjAlMjAlMjAlMjAwJTNBJTIwJTIyTyUyMiUyQyUwQSUyMCUyMCUyMCUyMDElM0ElMjAlMjJCLWNvcnBvcmF0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMiUzQSUyMCUyMkktY29ycG9yYXRpb24lMjIlMkMlMEElMjAlMjAlMjAlMjAzJTNBJTIwJTIyQi1jcmVhdGl2ZS13b3JrJTIyJTJDJTBBJTIwJTIwJTIwJTIwNCUzQSUyMCUyMkktY3JlYXRpdmUtd29yayUyMiUyQyUwQSUyMCUyMCUyMCUyMDUlM0ElMjAlMjJCLWdyb3VwJTIyJTJDJTBBJTIwJTIwJTIwJTIwNiUzQSUyMCUyMkktZ3JvdXAlMjIlMkMlMEElMjAlMjAlMjAlMjA3JTNBJTIwJTIyQi1sb2NhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDglM0ElMjAlMjJJLWxvY2F0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwOSUzQSUyMCUyMkItcGVyc29uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMTAlM0ElMjAlMjJJLXBlcnNvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDExJTNBJTIwJTIyQi1wcm9kdWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwMTIlM0ElMjAlMjJJLXByb2R1Y3QlMjIlMkMlMEElN0QlMEFsYWJlbDJpZCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMk8lMjIlM0ElMjAwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1jb3Jwb3JhdGlvbiUyMiUzQSUyMDElMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNvcnBvcmF0aW9uJTIyJTNBJTIwMiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItY3JlYXRpdmUtd29yayUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNyZWF0aXZlLXdvcmslMjIlM0ElMjA0JTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1ncm91cCUyMiUzQSUyMDUlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWdyb3VwJTIyJTNBJTIwNiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItbG9jYXRpb24lMjIlM0ElMjA3JTJDJTBBJTIwJTIwJTIwJTIwJTIySS1sb2NhdGlvbiUyMiUzQSUyMDglMkMlMEElMjAlMjAlMjAlMjAlMjJCLXBlcnNvbiUyMiUzQSUyMDklMkMlMEElMjAlMjAlMjAlMjAlMjJJLXBlcnNvbiUyMiUzQSUyMDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1wcm9kdWN0JTIyJTNBJTIwMTElMkMlMEElMjAlMjAlMjAlMjAlMjJJLXByb2R1Y3QlMjIlM0ElMjAxMiUyQyUwQSU3RA==",highlighted:`<span class="hljs-meta">>>> </span>id2label = {
	<span class="hljs-meta">... </span> <span class="hljs-number">0</span>: <span class="hljs-string">"O"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">1</span>: <span class="hljs-string">"B-corporation"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">2</span>: <span class="hljs-string">"I-corporation"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">3</span>: <span class="hljs-string">"B-creative-work"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">4</span>: <span class="hljs-string">"I-creative-work"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">5</span>: <span class="hljs-string">"B-group"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">6</span>: <span class="hljs-string">"I-group"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">7</span>: <span class="hljs-string">"B-location"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">8</span>: <span class="hljs-string">"I-location"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">9</span>: <span class="hljs-string">"B-person"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">10</span>: <span class="hljs-string">"I-person"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">11</span>: <span class="hljs-string">"B-product"</span>,
	<span class="hljs-meta">... </span> <span class="hljs-number">12</span>: <span class="hljs-string">"I-product"</span>,
	<span class="hljs-meta">... </span>}
	<span class="hljs-meta">>>> </span>label2id = {
	<span class="hljs-meta">... </span> <span class="hljs-string">"O"</span>: <span class="hljs-number">0</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"B-corporation"</span>: <span class="hljs-number">1</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"I-corporation"</span>: <span class="hljs-number">2</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"B-creative-work"</span>: <span class="hljs-number">3</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"I-creative-work"</span>: <span class="hljs-number">4</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"B-group"</span>: <span class="hljs-number">5</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"I-group"</span>: <span class="hljs-number">6</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"B-location"</span>: <span class="hljs-number">7</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"I-location"</span>: <span class="hljs-number">8</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"B-person"</span>: <span class="hljs-number">9</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"I-person"</span>: <span class="hljs-number">10</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"B-product"</span>: <span class="hljs-number">11</span>,
	<span class="hljs-meta">... </span> <span class="hljs-string">"I-product"</span>: <span class="hljs-number">12</span>,
	<span class="hljs-meta">... </span>}`,wrap:!1}}),ps=new Bl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[ka],pytorch:[Ua]},$$scope:{ctx:_}}}),rs=new Wl({props:{$$slots:{default:[_a]},$$scope:{ctx:_}}}),Es=new Ss({props:{title:"추론",local:"inference",headingTag:"h2"}}),zs=new k({props:{code:"dGV4dCUyMCUzRCUyMCUyMlRoZSUyMEdvbGRlbiUyMFN0YXRlJTIwV2FycmlvcnMlMjBhcmUlMjBhbiUyMEFtZXJpY2FuJTIwcHJvZmVzc2lvbmFsJTIwYmFza2V0YmFsbCUyMHRlYW0lMjBiYXNlZCUyMGluJTIwU2FuJTIwRnJhbmNpc2NvLiUyMg==",highlighted:'<span class="hljs-meta">>>> </span>text = <span class="hljs-string">"The Golden State Warriors are an American professional basketball team based in San Francisco."</span>',wrap:!1}}),Ns=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKCUyMm5lciUyMiUyQyUyMG1vZGVsJTNEJTIyc3RldmhsaXUlMkZteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIpJTBBY2xhc3NpZmllcih0ZXh0KQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

	<span class="hljs-meta">>>> </span>classifier = pipeline(<span class="hljs-string">"ner"</span>, model=<span class="hljs-string">"stevhliu/my_awesome_wnut_model"</span>)
	<span class="hljs-meta">>>> </span>classifier(text)
	[{<span class="hljs-string">'entity'</span>: <span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'score'</span>: <span class="hljs-number">0.42658573</span>,
	<span class="hljs-string">'index'</span>: <span class="hljs-number">2</span>,
	<span class="hljs-string">'word'</span>: <span class="hljs-string">'golden'</span>,
	<span class="hljs-string">'start'</span>: <span class="hljs-number">4</span>,
	<span class="hljs-string">'end'</span>: <span class="hljs-number">10</span>},
	{<span class="hljs-string">'entity'</span>: <span class="hljs-string">'I-location'</span>,
	<span class="hljs-string">'score'</span>: <span class="hljs-number">0.35856336</span>,
	<span class="hljs-string">'index'</span>: <span class="hljs-number">3</span>,
	<span class="hljs-string">'word'</span>: <span class="hljs-string">'state'</span>,
	<span class="hljs-string">'start'</span>: <span class="hljs-number">11</span>,
	<span class="hljs-string">'end'</span>: <span class="hljs-number">16</span>},
	{<span class="hljs-string">'entity'</span>: <span class="hljs-string">'B-group'</span>,
	<span class="hljs-string">'score'</span>: <span class="hljs-number">0.3064001</span>,
	<span class="hljs-string">'index'</span>: <span class="hljs-number">4</span>,
	<span class="hljs-string">'word'</span>: <span class="hljs-string">'warriors'</span>,
	<span class="hljs-string">'start'</span>: <span class="hljs-number">17</span>,
	<span class="hljs-string">'end'</span>: <span class="hljs-number">25</span>},
	{<span class="hljs-string">'entity'</span>: <span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'score'</span>: <span class="hljs-number">0.65523505</span>,
	<span class="hljs-string">'index'</span>: <span class="hljs-number">13</span>,
	<span class="hljs-string">'word'</span>: <span class="hljs-string">'san'</span>,
	<span class="hljs-string">'start'</span>: <span class="hljs-number">80</span>,
	<span class="hljs-string">'end'</span>: <span class="hljs-number">83</span>},
	{<span class="hljs-string">'entity'</span>: <span class="hljs-string">'B-location'</span>,
	<span class="hljs-string">'score'</span>: <span class="hljs-number">0.4668663</span>,
	<span class="hljs-string">'index'</span>: <span class="hljs-number">14</span>,
	<span class="hljs-string">'word'</span>: <span class="hljs-string">'francisco'</span>,
	<span class="hljs-string">'start'</span>: <span class="hljs-number">84</span>,
	<span class="hljs-string">'end'</span>: <span class="hljs-number">93</span>}]`,wrap:!1}}),is=new Bl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[va],pytorch:[Ia]},$$scope:{ctx:_}}}),qs=new ya({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/ko/tasks/token_classification.md"}}),{c(){a=b("meta"),o=i(),l=b("p"),m=i(),j(w.$$.fragment),C=i(),j(A.$$.fragment),v=i(),j(I.$$.fragment),G=i(),x=b("p"),x.textContent=V,W=i(),$=b("p"),$.textContent=B,r=i(),U=b("ol"),U.innerHTML=E,X=i(),j(R.$$.fragment),M=i(),Z=b("p"),Z.textContent=H,os=i(),j(F.$$.fragment),z=i(),Q=b("p"),Q.textContent=ls,ms=i(),j(N.$$.fragment),Y=i(),j(q.$$.fragment),S=i(),as=b("p"),as.textContent=Ms,L=i(),j(D.$$.fragment),P=i(),ts=b("p"),ts.textContent=js,K=i(),j(O.$$.fragment),ss=i(),ns=b("p"),ns.innerHTML=hs,p=i(),j(T.$$.fragment),us=i(),ys=b("p"),ys.innerHTML=Al,Ps=i(),fs=b("ul"),fs.innerHTML=Rl,Ks=i(),j(gs.$$.fragment),Os=i(),j(bs.$$.fragment),sl=i(),ds=b("p"),ds.innerHTML=Xl,ll=i(),j(ws.$$.fragment),al=i(),Js=b("p"),Js.innerHTML=Vl,tl=i(),j(Ts.$$.fragment),nl=i(),Us=b("p"),Us.innerHTML=El,el=i(),$s=b("ol"),$s.innerHTML=Hl,pl=i(),xs=b("p"),xs.textContent=Fl,rl=i(),j(ks.$$.fragment),il=i(),_s=b("p"),_s.innerHTML=zl,cl=i(),j(Cs.$$.fragment),ol=i(),Is=b("p"),Is.innerHTML=Ql,ml=i(),j(es.$$.fragment),Ml=i(),j(Zs.$$.fragment),jl=i(),vs=b("p"),vs.innerHTML=Nl,hl=i(),j(Gs.$$.fragment),ul=i(),Ws=b("p"),Ws.innerHTML=Yl,yl=i(),j(Bs.$$.fragment),fl=i(),As=b("p"),As.innerHTML=ql,gl=i(),j(Rs.$$.fragment),bl=i(),Xs=b("p"),Xs.innerHTML=Sl,dl=i(),j(Vs.$$.fragment),wl=i(),j(ps.$$.fragment),Jl=i(),j(rs.$$.fragment),Tl=i(),j(Es.$$.fragment),Ul=i(),Hs=b("p"),Hs.textContent=Ll,$l=i(),Fs=b("p"),Fs.textContent=Dl,xl=i(),j(zs.$$.fragment),kl=i(),Qs=b("p"),Qs.innerHTML=Pl,_l=i(),j(Ns.$$.fragment),Cl=i(),Ys=b("p"),Ys.innerHTML=Kl,Il=i(),j(is.$$.fragment),Zl=i(),j(qs.$$.fragment),vl=i(),Ds=b("p"),this.h()},l(s){const e=ma("svelte-u9bgzb",document.head);a=d(e,"META",{name:!0,content:!0}),e.forEach(t),o=c(s),l=d(s,"P",{}),na(l).forEach(t),m=c(s),h(w.$$.fragment,s),C=c(s),h(A.$$.fragment,s),v=c(s),h(I.$$.fragment,s),G=c(s),x=d(s,"P",{"data-svelte-h":!0}),J(x)!=="svelte-h22iel"&&(x.textContent=V),W=c(s),$=d(s,"P",{"data-svelte-h":!0}),J($)!=="svelte-14hiaa1"&&($.textContent=B),r=c(s),U=d(s,"OL",{"data-svelte-h":!0}),J(U)!=="svelte-kskchb"&&(U.innerHTML=E),X=c(s),h(R.$$.fragment,s),M=c(s),Z=d(s,"P",{"data-svelte-h":!0}),J(Z)!=="svelte-1bc8bfk"&&(Z.textContent=H),os=c(s),h(F.$$.fragment,s),z=c(s),Q=d(s,"P",{"data-svelte-h":!0}),J(Q)!=="svelte-1hhyu5y"&&(Q.textContent=ls),ms=c(s),h(N.$$.fragment,s),Y=c(s),h(q.$$.fragment,s),S=c(s),as=d(s,"P",{"data-svelte-h":!0}),J(as)!=="svelte-1v008tf"&&(as.textContent=Ms),L=c(s),h(D.$$.fragment,s),P=c(s),ts=d(s,"P",{"data-svelte-h":!0}),J(ts)!=="svelte-7y00sm"&&(ts.textContent=js),K=c(s),h(O.$$.fragment,s),ss=c(s),ns=d(s,"P",{"data-svelte-h":!0}),J(ns)!=="svelte-19kj2kd"&&(ns.innerHTML=hs),p=c(s),h(T.$$.fragment,s),us=c(s),ys=d(s,"P",{"data-svelte-h":!0}),J(ys)!=="svelte-1jetaks"&&(ys.innerHTML=Al),Ps=c(s),fs=d(s,"UL",{"data-svelte-h":!0}),J(fs)!=="svelte-wzon1c"&&(fs.innerHTML=Rl),Ks=c(s),h(gs.$$.fragment,s),Os=c(s),h(bs.$$.fragment,s),sl=c(s),ds=d(s,"P",{"data-svelte-h":!0}),J(ds)!=="svelte-xprzxq"&&(ds.innerHTML=Xl),ll=c(s),h(ws.$$.fragment,s),al=c(s),Js=d(s,"P",{"data-svelte-h":!0}),J(Js)!=="svelte-b47600"&&(Js.innerHTML=Vl),tl=c(s),h(Ts.$$.fragment,s),nl=c(s),Us=d(s,"P",{"data-svelte-h":!0}),J(Us)!=="svelte-1slve6o"&&(Us.innerHTML=El),el=c(s),$s=d(s,"OL",{"data-svelte-h":!0}),J($s)!=="svelte-1ynfql3"&&($s.innerHTML=Hl),pl=c(s),xs=d(s,"P",{"data-svelte-h":!0}),J(xs)!=="svelte-1uatru5"&&(xs.textContent=Fl),rl=c(s),h(ks.$$.fragment,s),il=c(s),_s=d(s,"P",{"data-svelte-h":!0}),J(_s)!=="svelte-o2pm0z"&&(_s.innerHTML=zl),cl=c(s),h(Cs.$$.fragment,s),ol=c(s),Is=d(s,"P",{"data-svelte-h":!0}),J(Is)!=="svelte-19lacyh"&&(Is.innerHTML=Ql),ml=c(s),h(es.$$.fragment,s),Ml=c(s),h(Zs.$$.fragment,s),jl=c(s),vs=d(s,"P",{"data-svelte-h":!0}),J(vs)!=="svelte-1xei3jh"&&(vs.innerHTML=Nl),hl=c(s),h(Gs.$$.fragment,s),ul=c(s),Ws=d(s,"P",{"data-svelte-h":!0}),J(Ws)!=="svelte-no1q5"&&(Ws.innerHTML=Yl),yl=c(s),h(Bs.$$.fragment,s),fl=c(s),As=d(s,"P",{"data-svelte-h":!0}),J(As)!=="svelte-1tfmmd0"&&(As.innerHTML=ql),gl=c(s),h(Rs.$$.fragment,s),bl=c(s),Xs=d(s,"P",{"data-svelte-h":!0}),J(Xs)!=="svelte-84b3vk"&&(Xs.innerHTML=Sl),dl=c(s),h(Vs.$$.fragment,s),wl=c(s),h(ps.$$.fragment,s),Jl=c(s),h(rs.$$.fragment,s),Tl=c(s),h(Es.$$.fragment,s),Ul=c(s),Hs=d(s,"P",{"data-svelte-h":!0}),J(Hs)!=="svelte-1r6hgyn"&&(Hs.textContent=Ll),$l=c(s),Fs=d(s,"P",{"data-svelte-h":!0}),J(Fs)!=="svelte-ej1eir"&&(Fs.textContent=Dl),xl=c(s),h(zs.$$.fragment,s),kl=c(s),Qs=d(s,"P",{"data-svelte-h":!0}),J(Qs)!=="svelte-bavprk"&&(Qs.innerHTML=Pl),_l=c(s),h(Ns.$$.fragment,s),Cl=c(s),Ys=d(s,"P",{"data-svelte-h":!0}),J(Ys)!=="svelte-1smgeha"&&(Ys.innerHTML=Kl),Il=c(s),h(is.$$.fragment,s),Zl=c(s),h(qs.$$.fragment,s),vl=c(s),Ds=d(s,"P",{}),na(Ds).forEach(t),this.h()},h(){ea(a,"name","hf:doc:metadata"),ea(a,"content",Wa)},m(s,e){Ma(document.head,a),n(s,o,e),n(s,l,e),n(s,m,e),u(w,s,e),n(s,C,e),u(A,s,e),n(s,v,e),u(I,s,e),n(s,G,e),n(s,x,e),n(s,W,e),n(s,$,e),n(s,r,e),n(s,U,e),n(s,X,e),u(R,s,e),n(s,M,e),n(s,Z,e),n(s,os,e),u(F,s,e),n(s,z,e),n(s,Q,e),n(s,ms,e),u(N,s,e),n(s,Y,e),u(q,s,e),n(s,S,e),n(s,as,e),n(s,L,e),u(D,s,e),n(s,P,e),n(s,ts,e),n(s,K,e),u(O,s,e),n(s,ss,e),n(s,ns,e),n(s,p,e),u(T,s,e),n(s,us,e),n(s,ys,e),n(s,Ps,e),n(s,fs,e),n(s,Ks,e),u(gs,s,e),n(s,Os,e),u(bs,s,e),n(s,sl,e),n(s,ds,e),n(s,ll,e),u(ws,s,e),n(s,al,e),n(s,Js,e),n(s,tl,e),u(Ts,s,e),n(s,nl,e),n(s,Us,e),n(s,el,e),n(s,$s,e),n(s,pl,e),n(s,xs,e),n(s,rl,e),u(ks,s,e),n(s,il,e),n(s,_s,e),n(s,cl,e),u(Cs,s,e),n(s,ol,e),n(s,Is,e),n(s,ml,e),u(es,s,e),n(s,Ml,e),u(Zs,s,e),n(s,jl,e),n(s,vs,e),n(s,hl,e),u(Gs,s,e),n(s,ul,e),n(s,Ws,e),n(s,yl,e),u(Bs,s,e),n(s,fl,e),n(s,As,e),n(s,gl,e),u(Rs,s,e),n(s,bl,e),n(s,Xs,e),n(s,dl,e),u(Vs,s,e),n(s,wl,e),u(ps,s,e),n(s,Jl,e),u(rs,s,e),n(s,Tl,e),u(Es,s,e),n(s,Ul,e),n(s,Hs,e),n(s,$l,e),n(s,Fs,e),n(s,xl,e),u(zs,s,e),n(s,kl,e),n(s,Qs,e),n(s,_l,e),u(Ns,s,e),n(s,Cl,e),n(s,Ys,e),n(s,Il,e),u(is,s,e),n(s,Zl,e),u(qs,s,e),n(s,vl,e),n(s,Ds,e),Gl=!0},p(s,[e]){const Ol={};e&2&&(Ol.$$scope={dirty:e,ctx:s}),R.$set(Ol);const sa={};e&2&&(sa.$$scope={dirty:e,ctx:s}),es.$set(sa);const la={};e&2&&(la.$$scope={dirty:e,ctx:s}),ps.$set(la);const aa={};e&2&&(aa.$$scope={dirty:e,ctx:s}),rs.$set(aa);const ta={};e&2&&(ta.$$scope={dirty:e,ctx:s}),is.$set(ta)},i(s){Gl\|\|(y(w.$$.fragment,s),y(A.$$.fragment,s),y(I.$$.fragment,s),y(R.$$.fragment,s),y(F.$$.fragment,s),y(N.$$.fragment,s),y(q.$$.fragment,s),y(D.$$.fragment,s),y(O.$$.fragment,s),y(T.$$.fragment,s),y(gs.$$.fragment,s),y(bs.$$.fragment,s),y(ws.$$.fragment,s),y(Ts.$$.fragment,s),y(ks.$$.fragment,s),y(Cs.$$.fragment,s),y(es.$$.fragment,s),y(Zs.$$.fragment,s),y(Gs.$$.fragment,s),y(Bs.$$.fragment,s),y(Rs.$$.fragment,s),y(Vs.$$.fragment,s),y(ps.$$.fragment,s),y(rs.$$.fragment,s),y(Es.$$.fragment,s),y(zs.$$.fragment,s),y(Ns.$$.fragment,s),y(is.$$.fragment,s),y(qs.$$.fragment,s),Gl=!0)},o(s){f(w.$$.fragment,s),f(A.$$.fragment,s),f(I.$$.fragment,s),f(R.$$.fragment,s),f(F.$$.fragment,s),f(N.$$.fragment,s),f(q.$$.fragment,s),f(D.$$.fragment,s),f(O.$$.fragment,s),f(T.$$.fragment,s),f(gs.$$.fragment,s),f(bs.$$.fragment,s),f(ws.$$.fragment,s),f(Ts.$$.fragment,s),f(ks.$$.fragment,s),f(Cs.$$.fragment,s),f(es.$$.fragment,s),f(Zs.$$.fragment,s),f(Gs.$$.fragment,s),f(Bs.$$.fragment,s),f(Rs.$$.fragment,s),f(Vs.$$.fragment,s),f(ps.$$.fragment,s),f(rs.$$.fragment,s),f(Es.$$.fragment,s),f(zs.$$.fragment,s),f(Ns.$$.fragment,s),f(is.$$.fragment,s),f(qs.$$.fragment,s),Gl=!1},d(s){s&&(t(o),t(l),t(m),t(C),t(v),t(G),t(x),t(W),t($),t(r),t(U),t(X),t(M),t(Z),t(os),t(z),t(Q),t(ms),t(Y),t(S),t(as),t(L),t(P),t(ts),t(K),t(ss),t(ns),t(p),t(us),t(ys),t(Ps),t(fs),t(Ks),t(Os),t(sl),t(ds),t(ll),t(al),t(Js),t(tl),t(nl),t(Us),t(el),t($s),t(pl),t(xs),t(rl),t(il),t(_s),t(cl),t(ol),t(Is),t(ml),t(Ml),t(jl),t(vs),t(hl),t(ul),t(Ws),t(yl),t(fl),t(As),t(gl),t(bl),t(Xs),t(dl),t(wl),t(Jl),t(Tl),t(Ul),t(Hs),t($l),t(Fs),t(xl),t(kl),t(Qs),t(_l),t(Cl),t(Ys),t(Il),t(Zl),t(vl),t(Ds)),t(a),g(w,s),g(A,s),g(I,s),g(R,s),g(F,s),g(N,s),g(q,s),g(D,s),g(O,s),g(T,s),g(gs,s),g(bs,s),g(ws,s),g(Ts,s),g(ks,s),g(Cs,s),g(es,s),g(Zs,s),g(Gs,s),g(Bs,s),g(Rs,s),g(Vs,s),g(ps,s),g(rs,s),g(Es,s),g(zs,s),g(Ns,s),g(is,s),g(qs,s)}}}const Wa='{"title":"토큰 분류","local":"token-classification","sections":[{"title":"WNUT 17 데이터 세트 가져오기","local":"load-wnut-17-dataset","sections":[],"depth":2},{"title":"전처리","local":"preprocess","sections":[],"depth":2},{"title":"평가","local":"evaluation","sections":[],"depth":2},{"title":"훈련","local":"train","sections":[],"depth":2},{"title":"추론","local":"inference","sections":[],"depth":2}],"depth":1}';function Ba(_){return ia(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Qa extends ca{constructor(a){super(),oa(this,a,Ba,Ga,ra,{})}}export{Qa as component};

Xet Storage Details

Size:: 76.7 kB
Xet hash:: 409c5be6c8de740199a71706811176d3d1f5e788218961fbde9ae8e0c5074a86

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.