course/pr_1021/fr/_app/immutable/nodes/52.d596aaa5.js

import{s as F,n as G,o as N}from"../chunks/scheduler.37c15a92.js";import{S as O,i as R,g as f,s as a,r as w,A as W,h as d,f as n,c as s,j as U,u as C,x as E,k as j,y as D,a as i,v as H,d as L,t as M,w as y}from"../chunks/index.2bf4358c.js";import{C as I}from"../chunks/CourseFloatingBanner.6add7356.js";import{H as J,E as K}from"../chunks/getInferenceSnippets.ebf8be91.js";function Q(q){let r,_,h,b,o,g,l,v,m,A="Bon travail pour finir ce chapitre !",z,c,B="Après cette plongée en profondeur dans les <em>tokenizers</em>, vous devriez :",k,p,S="<li>être capable d’entraîner un nouveau tokenizer en utilisant un ancien tokenizer comme modèle,</li> <li>comprendre comment utiliser les <em>offsets</em> pour faire correspondre la position des <em>tokens</em> à l’étendue de texte d’origine,</li> <li>connaître les différences entre BPE, <em>WordPiece</em> et <em>Unigram</em>,</li> <li>être capable de combiner les blocs fournis par la bibliothèque 🤗 <em>Tokenizers</em> pour construire votre propre <em>tokenizer</em>,</li> <li>être capable d’utiliser ce <em>tokenizer</em> dans la bibliothèque 🤗 <em>Transformers</em>.</li>",x,u,P,$,T;return o=new J({props:{title:"<i> Tokenizer </i> , coché !",local:"i-tokenizer-i--coché-",headingTag:"h1"}}),l=new I({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),u=new K({props:{source:"https://github.com/huggingface/course/blob/main/chapters/fr/chapter6/9.mdx"}}),{c(){r=f("meta"),_=a(),h=f("p"),b=a(),w(o.$$.fragment),g=a(),w(l.$$.fragment),v=a(),m=f("p"),m.textContent=A,z=a(),c=f("p"),c.innerHTML=B,k=a(),p=f("ul"),p.innerHTML=S,x=a(),w(u.$$.fragment),P=a(),$=f("p"),this.h()},l(e){const t=W("svelte-u9bgzb",document.head);r=d(t,"META",{name:!0,content:!0}),t.forEach(n),_=s(e),h=d(e,"P",{}),U(h).forEach(n),b=s(e),C(o.$$.fragment,e),g=s(e),C(l.$$.fragment,e),v=s(e),m=d(e,"P",{"data-svelte-h":!0}),E(m)!=="svelte-1nu3oa1"&&(m.textContent=A),z=s(e),c=d(e,"P",{"data-svelte-h":!0}),E(c)!=="svelte-thw73"&&(c.innerHTML=B),k=s(e),p=d(e,"UL",{"data-svelte-h":!0}),E(p)!=="svelte-16sd9em"&&(p.innerHTML=S),x=s(e),C(u.$$.fragment,e),P=s(e),$=d(e,"P",{}),U($).forEach(n),this.h()},h(){j(r,"name","hf:doc:metadata"),j(r,"content",V)},m(e,t){D(document.head,r),i(e,_,t),i(e,h,t),i(e,b,t),H(o,e,t),i(e,g,t),H(l,e,t),i(e,v,t),i(e,m,t),i(e,z,t),i(e,c,t),i(e,k,t),i(e,p,t),i(e,x,t),H(u,e,t),i(e,P,t),i(e,$,t),T=!0},p:G,i(e){T||(L(o.$$.fragment,e),L(l.$$.fragment,e),L(u.$$.fragment,e),T=!0)},o(e){M(o.$$.fragment,e),M(l.$$.fragment,e),M(u.$$.fragment,e),T=!1},d(e){e&&(n(_),n(h),n(b),n(g),n(v),n(m),n(z),n(c),n(k),n(p),n(x),n(P),n($)),n(r),y(o,e),y(l,e),y(u,e)}}}const V='{"title":"<i> Tokenizer </i> , coché !","local":"i-tokenizer-i--coché-","sections":[],"depth":1}';function X(q){return N(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ne extends O{constructor(r){super(),R(this,r,X,Q,F,{})}}export{ne as component};