tokenizers/pr_2139/en/_app/immutable/nodes/16.b006fe2c.js

import{s as ft,o as mt,n as nt}from"../chunks/scheduler.7c59faff.js";import{S as ut,i as gt,e as k,s as u,c as v,h as ht,a as y,d as a,b as g,f as $t,g as b,j as pt,k as ct,l as i,m as _,t as T,n as x,o as C,p as H}from"../chunks/index.09bb5655.js";import{C as dt,H as E,E as wt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fb0d6fe2.js";import{C as P}from"../chunks/CodeBlock.aaa4219f.js";import{T as vt,M as lt}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";function bt(M){let n,p="🤗 Tokenizers is tested on Python 3.5+.",e,o,m=`You should install 🤗 Tokenizers in a <a href="https://docs.python.org/3/library/venv.html" rel="nofollow">virtual environment</a>. If you’re
unfamiliar with Python virtual environments, check out the <a href="https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/" rel="nofollow">user
guide</a>.
Create a virtual environment with the version of Python you’re going to
use and activate it.`,c,d,r,f,z="🤗 Tokenizers can be installed using pip as follows:",s,w,I,J,l,h,V=`To use this method, you need to have the Rust language installed. You
can follow <a href="https://www.rust-lang.org/learn/get-started" rel="nofollow">the official
guide</a> for more
information.`,F,L,at=`If you are using a unix based OS, the installation should be as simple
as running:`,W,j,N,G,it="Or you can easily update it with the following command:",O,R,q,Z,st=`Once rust is installed, we can start retrieving the sources for 🤗
Tokenizers:`,A,U,Q,Y,rt="Then we go into the python bindings folder:",D,S,K,B,ot=`At this point you should have your <a href="">virtual environment</a> already
activated. In order to compile 🤗 Tokenizers, you need to:`,tt,X,et;return d=new E({props:{title:"Installation with pip",local:"installation-with-pip",headingTag:"h2"}}),w=new P({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRva2VuaXplcnM=",highlighted:"pip install tokenizers",lang:"bash",wrap:!1}}),J=new E({props:{title:"Installation from sources",local:"installation-from-sources",headingTag:"h2"}}),j=new P({props:{code:"Y3VybCUyMC0tcHJvdG8lMjAnJTNEaHR0cHMnJTIwLS10bHN2MS4yJTIwLXNTZiUyMGh0dHBzJTNBJTJGJTJGc2gucnVzdHVwLnJzJTIwJTdDJTIwc2g=",highlighted:'curl --proto <span class="hljs-string">&#x27;=https&#x27;</span> --tlsv1.2 -sSf https://sh.rustup.rs | sh',lang:"bash",wrap:!1}}),R=new P({props:{code:"cnVzdHVwJTIwdXBkYXRl",highlighted:"rustup update",lang:"bash",wrap:!1}}),U=new P({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRnRva2VuaXplcnM=",highlighted:'git <span class="hljs-built_in">clone</span> https://github.com/huggingface/tokenizers',lang:"bash",wrap:!1}}),S=new P({props:{code:"Y2QlMjB0b2tlbml6ZXJzJTJGYmluZGluZ3MlMkZweXRob24=",highlighted:'<span class="hljs-built_in">cd</span> tokenizers/bindings/python',lang:"bash",wrap:!1}}),X=new P({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1lJTIwLg==",highlighted:"pip install -e .",lang:"bash",wrap:!1}}),{c(){n=k("p"),n.textContent=p,e=u(),o=k("p"),o.innerHTML=m,c=u(),v(d.$$.fragment),r=u(),f=k("p"),f.textContent=z,s=u(),v(w.$$.fragment),I=u(),v(J.$$.fragment),l=u(),h=k("p"),h.innerHTML=V,F=u(),L=k("p"),L.textContent=at,W=u(),v(j.$$.fragment),N=u(),G=k("p"),G.textContent=it,O=u(),v(R.$$.fragment),q=u(),Z=k("p"),Z.textContent=st,A=u(),v(U.$$.fragment),Q=u(),Y=k("p"),Y.textContent=rt,D=u(),v(S.$$.fragment),K=u(),B=k("p"),B.innerHTML=ot,tt=u(),v(X.$$.fragment)},l(t){n=y(t,"P",{"data-svelte-h":!0}),H(n)!=="svelte-fve8i2"&&(n.textContent=p),e=g(t),o=y(t,"P",{"data-svelte-h":!0}),H(o)!=="svelte-lcd1tm"&&(o.innerHTML=m),c=g(t),b(d.$$.fragment,t),r=g(t),f=y(t,"P",{"data-svelte-h":!0}),H(f)!=="svelte-1xhb6ej"&&(f.textContent=z),s=g(t),b(w.$$.fragment,t),I=g(t),b(J.$$.fragment,t),l=g(t),h=y(t,"P",{"data-svelte-h":!0}),H(h)!=="svelte-u0q76u"&&(h.innerHTML=V),F=g(t),L=y(t,"P",{"data-svelte-h":!0}),H(L)!=="svelte-1ipuuw6"&&(L.textContent=at),W=g(t),b(j.$$.fragment,t),N=g(t),G=y(t,"P",{"data-svelte-h":!0}),H(G)!=="svelte-7tnt7y"&&(G.textContent=it),O=g(t),b(R.$$.fragment,t),q=g(t),Z=y(t,"P",{"data-svelte-h":!0}),H(Z)!=="svelte-1pytytn"&&(Z.textContent=st),A=g(t),b(U.$$.fragment,t),Q=g(t),Y=y(t,"P",{"data-svelte-h":!0}),H(Y)!=="svelte-10nv0ss"&&(Y.textContent=rt),D=g(t),b(S.$$.fragment,t),K=g(t),B=y(t,"P",{"data-svelte-h":!0}),H(B)!=="svelte-ktt0vl"&&(B.innerHTML=ot),tt=g(t),b(X.$$.fragment,t)},m(t,$){i(t,n,$),i(t,e,$),i(t,o,$),i(t,c,$),_(d,t,$),i(t,r,$),i(t,f,$),i(t,s,$),_(w,t,$),i(t,I,$),_(J,t,$),i(t,l,$),i(t,h,$),i(t,F,$),i(t,L,$),i(t,W,$),_(j,t,$),i(t,N,$),i(t,G,$),i(t,O,$),_(R,t,$),i(t,q,$),i(t,Z,$),i(t,A,$),_(U,t,$),i(t,Q,$),i(t,Y,$),i(t,D,$),_(S,t,$),i(t,K,$),i(t,B,$),i(t,tt,$),_(X,t,$),et=!0},p:nt,i(t){et||(T(d.$$.fragment,t),T(w.$$.fragment,t),T(J.$$.fragment,t),T(j.$$.fragment,t),T(R.$$.fragment,t),T(U.$$.fragment,t),T(S.$$.fragment,t),T(X.$$.fragment,t),et=!0)},o(t){x(d.$$.fragment,t),x(w.$$.fragment,t),x(J.$$.fragment,t),x(j.$$.fragment,t),x(R.$$.fragment,t),x(U.$$.fragment,t),x(S.$$.fragment,t),x(X.$$.fragment,t),et=!1},d(t){t&&(a(n),a(e),a(o),a(c),a(r),a(f),a(s),a(I),a(l),a(h),a(F),a(L),a(W),a(N),a(G),a(O),a(q),a(Z),a(A),a(Q),a(Y),a(D),a(K),a(B),a(tt)),C(d,t),C(w,t),C(J,t),C(j,t),C(R,t),C(U,t),C(S,t),C(X,t)}}}function _t(M){let n,p;return n=new lt({props:{$$slots:{default:[bt]},$$scope:{ctx:M}}}),{c(){v(n.$$.fragment)},l(e){b(n.$$.fragment,e)},m(e,o){_(n,e,o),p=!0},p(e,o){const m={};o&2&&(m.$$scope={dirty:o,ctx:e}),n.$set(m)},i(e){p||(T(n.$$.fragment,e),p=!0)},o(e){x(n.$$.fragment,e),p=!1},d(e){C(n,e)}}}function Tt(M){let n,p,e,o='🤗 Tokenizers is available on <a href="https://crates.io/crates/tokenizers" rel="nofollow">crates.io</a>.',m,c,d="You just need to add it to your <code>Cargo.toml</code>:",r,f,z;return n=new E({props:{title:"Crates.io",local:"cratesio",headingTag:"h2"}}),f=new P({props:{code:"Y2FyZ28lMjBhZGQlMjB0b2tlbml6ZXJz",highlighted:"cargo add tokenizers",lang:"bash",wrap:!1}}),{c(){v(n.$$.fragment),p=u(),e=k("p"),e.innerHTML=o,m=u(),c=k("p"),c.innerHTML=d,r=u(),v(f.$$.fragment)},l(s){b(n.$$.fragment,s),p=g(s),e=y(s,"P",{"data-svelte-h":!0}),H(e)!=="svelte-1apllmp"&&(e.innerHTML=o),m=g(s),c=y(s,"P",{"data-svelte-h":!0}),H(c)!=="svelte-gj78vj"&&(c.innerHTML=d),r=g(s),b(f.$$.fragment,s)},m(s,w){_(n,s,w),i(s,p,w),i(s,e,w),i(s,m,w),i(s,c,w),i(s,r,w),_(f,s,w),z=!0},p:nt,i(s){z||(T(n.$$.fragment,s),T(f.$$.fragment,s),z=!0)},o(s){x(n.$$.fragment,s),x(f.$$.fragment,s),z=!1},d(s){s&&(a(p),a(e),a(m),a(c),a(r)),C(n,s),C(f,s)}}}function xt(M){let n,p;return n=new lt({props:{$$slots:{default:[Tt]},$$scope:{ctx:M}}}),{c(){v(n.$$.fragment)},l(e){b(n.$$.fragment,e)},m(e,o){_(n,e,o),p=!0},p(e,o){const m={};o&2&&(m.$$scope={dirty:o,ctx:e}),n.$set(m)},i(e){p||(T(n.$$.fragment,e),p=!0)},o(e){x(n.$$.fragment,e),p=!1},d(e){C(n,e)}}}function Ct(M){let n,p,e,o="You can simply install 🤗 Tokenizers with npm using:",m,c,d;return n=new E({props:{title:"Installation with npm",local:"installation-with-npm",headingTag:"h2"}}),c=new P({props:{code:"bnBtJTIwaW5zdGFsbCUyMHRva2VuaXplcnM=",highlighted:"npm install tokenizers",lang:"bash",wrap:!1}}),{c(){v(n.$$.fragment),p=u(),e=k("p"),e.textContent=o,m=u(),v(c.$$.fragment)},l(r){b(n.$$.fragment,r),p=g(r),e=y(r,"P",{"data-svelte-h":!0}),H(e)!=="svelte-1aawqxk"&&(e.textContent=o),m=g(r),b(c.$$.fragment,r)},m(r,f){_(n,r,f),i(r,p,f),i(r,e,f),i(r,m,f),_(c,r,f),d=!0},p:nt,i(r){d||(T(n.$$.fragment,r),T(c.$$.fragment,r),d=!0)},o(r){x(n.$$.fragment,r),x(c.$$.fragment,r),d=!1},d(r){r&&(a(p),a(e),a(m)),C(n,r),C(c,r)}}}function kt(M){let n,p;return n=new lt({props:{$$slots:{default:[Ct]},$$scope:{ctx:M}}}),{c(){v(n.$$.fragment)},l(e){b(n.$$.fragment,e)},m(e,o){_(n,e,o),p=!0},p(e,o){const m={};o&2&&(m.$$scope={dirty:o,ctx:e}),n.$set(m)},i(e){p||(T(n.$$.fragment,e),p=!0)},o(e){x(n.$$.fragment,e),p=!1},d(e){C(n,e)}}}function yt(M){let n,p,e,o,m,c,d,r,f,z,s,w,I,J;return m=new dt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),d=new E({props:{title:"Installation",local:"installation",headingTag:"h1"}}),f=new vt({props:{python:!0,rust:!0,node:!0,$$slots:{node:[kt],rust:[xt],python:[_t]},$$scope:{ctx:M}}}),s=new wt({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/installation.mdx"}}),{c(){n=k("meta"),p=u(),e=k("p"),o=u(),v(m.$$.fragment),c=u(),v(d.$$.fragment),r=u(),v(f.$$.fragment),z=u(),v(s.$$.fragment),w=u(),I=k("p"),this.h()},l(l){const h=ht("svelte-u9bgzb",document.head);n=y(h,"META",{name:!0,content:!0}),h.forEach(a),p=g(l),e=y(l,"P",{}),$t(e).forEach(a),o=g(l),b(m.$$.fragment,l),c=g(l),b(d.$$.fragment,l),r=g(l),b(f.$$.fragment,l),z=g(l),b(s.$$.fragment,l),w=g(l),I=y(l,"P",{}),$t(I).forEach(a),this.h()},h(){pt(n,"name","hf:doc:metadata"),pt(n,"content",Mt)},m(l,h){ct(document.head,n),i(l,p,h),i(l,e,h),i(l,o,h),_(m,l,h),i(l,c,h),_(d,l,h),i(l,r,h),_(f,l,h),i(l,z,h),_(s,l,h),i(l,w,h),i(l,I,h),J=!0},p(l,[h]){const V={};h&2&&(V.$$scope={dirty:h,ctx:l}),f.$set(V)},i(l){J||(T(m.$$.fragment,l),T(d.$$.fragment,l),T(f.$$.fragment,l),T(s.$$.fragment,l),J=!0)},o(l){x(m.$$.fragment,l),x(d.$$.fragment,l),x(f.$$.fragment,l),x(s.$$.fragment,l),J=!1},d(l){l&&(a(p),a(e),a(o),a(c),a(r),a(z),a(w),a(I)),a(n),C(m,l),C(d,l),C(f,l),C(s,l)}}}const Mt='{"title":"Installation","local":"installation","sections":[{"title":"Installation with pip","local":"installation-with-pip","sections":[],"depth":2},{"title":"Installation from sources","local":"installation-from-sources","sections":[],"depth":2},{"title":"Crates.io","local":"cratesio","sections":[],"depth":2},{"title":"Installation with npm","local":"installation-with-npm","sections":[],"depth":2}],"depth":1}';function zt(M){return mt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jt extends ut{constructor(n){super(),gt(this,n,zt,yt,ft,{})}}export{jt as component};