course/pr_1213/fr/_app/immutable/nodes/52.b320ca90.js

import{s as N,n as O,o as R}from"../chunks/scheduler.893fe8c9.js";import{S as W,i as D,e as $,s as a,c as L,h as I,a as d,d as n,b as s,f as F,g as M,j as q,k as G,l as J,m as i,n as y,t as E,o as H,p as S}from"../chunks/index.2d09ebb4.js";import{C as K,H as Q,E as V}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.f3ce5e15.js";import{C as X}from"../chunks/CourseFloatingBanner.2900b001.js";function Y(B){let r,_,h,g,o,v,l,x,m,z,p,U="Bon travail pour finir ce chapitre !",k,u,A="Après cette plongée en profondeur dans les <em>tokenizers</em>, vous devriez :",T,c,j="<li>être capable d’entraîner un nouveau tokenizer en utilisant un ancien tokenizer comme modèle,</li> <li>comprendre comment utiliser les <em>offsets</em> pour faire correspondre la position des <em>tokens</em> à l’étendue de texte d’origine,</li> <li>connaître les différences entre BPE, <em>WordPiece</em> et <em>Unigram</em>,</li> <li>être capable de combiner les blocs fournis par la bibliothèque 🤗 <em>Tokenizers</em> pour construire votre propre <em>tokenizer</em>,</li> <li>être capable d’utiliser ce <em>tokenizer</em> dans la bibliothèque 🤗 <em>Transformers</em>.</li>",C,f,P,b,w;return o=new K({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),l=new Q({props:{title:"<i> Tokenizer </i> , coché !",local:"i-tokenizer-i--coché-",headingTag:"h1"}}),m=new X({props:{chapter:6,classNames:"absolute z-10 right-0 top-0"}}),f=new V({props:{source:"https://github.com/huggingface/course/blob/main/chapters/fr/chapter6/9.mdx"}}),{c(){r=$("meta"),_=a(),h=$("p"),g=a(),L(o.$$.fragment),v=a(),L(l.$$.fragment),x=a(),L(m.$$.fragment),z=a(),p=$("p"),p.textContent=U,k=a(),u=$("p"),u.innerHTML=A,T=a(),c=$("ul"),c.innerHTML=j,C=a(),L(f.$$.fragment),P=a(),b=$("p"),this.h()},l(e){const t=I("svelte-u9bgzb",document.head);r=d(t,"META",{name:!0,content:!0}),t.forEach(n),_=s(e),h=d(e,"P",{}),F(h).forEach(n),g=s(e),M(o.$$.fragment,e),v=s(e),M(l.$$.fragment,e),x=s(e),M(m.$$.fragment,e),z=s(e),p=d(e,"P",{"data-svelte-h":!0}),q(p)!=="svelte-1nu3oa1"&&(p.textContent=U),k=s(e),u=d(e,"P",{"data-svelte-h":!0}),q(u)!=="svelte-thw73"&&(u.innerHTML=A),T=s(e),c=d(e,"UL",{"data-svelte-h":!0}),q(c)!=="svelte-16sd9em"&&(c.innerHTML=j),C=s(e),M(f.$$.fragment,e),P=s(e),b=d(e,"P",{}),F(b).forEach(n),this.h()},h(){G(r,"name","hf:doc:metadata"),G(r,"content",Z)},m(e,t){J(document.head,r),i(e,_,t),i(e,h,t),i(e,g,t),y(o,e,t),i(e,v,t),y(l,e,t),i(e,x,t),y(m,e,t),i(e,z,t),i(e,p,t),i(e,k,t),i(e,u,t),i(e,T,t),i(e,c,t),i(e,C,t),y(f,e,t),i(e,P,t),i(e,b,t),w=!0},p:O,i(e){w||(E(o.$$.fragment,e),E(l.$$.fragment,e),E(m.$$.fragment,e),E(f.$$.fragment,e),w=!0)},o(e){H(o.$$.fragment,e),H(l.$$.fragment,e),H(m.$$.fragment,e),H(f.$$.fragment,e),w=!1},d(e){e&&(n(_),n(h),n(g),n(v),n(x),n(z),n(p),n(k),n(u),n(T),n(c),n(C),n(P),n(b)),n(r),S(o,e),S(l,e),S(m,e),S(f,e)}}}const Z='{"title":"<i> Tokenizer </i> , coché !","local":"i-tokenizer-i--coché-","sections":[],"depth":1}';function ee(B){return R(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ae extends W{constructor(r){super(),D(this,r,ee,Y,N,{})}}export{ae as component};