tokenizers/pr_2139/en/_app/immutable/nodes/16.roMIeydn.js

import"../chunks/DsnmJJEf.js";import{i as I,h as C,C as G,H as r,E as $,s as H,a as n}from"../chunks/DPtJuB9X.js";import{p as P,o as R,s as a,f as d,a as h,b as Z,c as p,n as j}from"../chunks/CFwIJwKg.js";import{T as x,M as c}from"../chunks/Dd6Po5Jz.js";const U='{"title":"Installation","local":"installation","sections":[{"title":"Installation with pip","local":"installation-with-pip","sections":[],"depth":2},{"title":"Installation from sources","local":"installation-from-sources","sections":[],"depth":2},{"title":"Crates.io","local":"cratesio","sections":[],"depth":2},{"title":"Installation with npm","local":"installation-with-npm","sections":[],"depth":2}],"depth":1}';var Y=p('<meta name="hf:doc:metadata"/>'),B=p(`<p>🤗 Tokenizers is tested on Python 3.5+.</p> <p>You should install 🤗 Tokenizers in a <a href="https://docs.python.org/3/library/venv.html" rel="nofollow">virtual environment</a>. If you’re
unfamiliar with Python virtual environments, check out the <a href="https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/" rel="nofollow">user
guide</a>.
Create a virtual environment with the version of Python you’re going to
use and activate it.</p> <!> <p>🤗 Tokenizers can be installed using pip as follows:</p> <!> <!> <p>To use this method, you need to have the Rust language installed. You
can follow <a href="https://www.rust-lang.org/learn/get-started" rel="nofollow">the official
guide</a> for more
information.</p> <p>If you are using a unix based OS, the installation should be as simple
as running:</p> <!> <p>Or you can easily update it with the following command:</p> <!> <p>Once rust is installed, we can start retrieving the sources for 🤗
Tokenizers:</p> <!> <p>Then we go into the python bindings folder:</p> <!> <p>At this point you should have your <a href="">virtual environment</a> already
activated. In order to compile 🤗 Tokenizers, you need to:</p> <!>`,1),X=p('<!> <p>🤗 Tokenizers is available on <a href="https://crates.io/crates/tokenizers" rel="nofollow">crates.io</a>.</p> <p>You just need to add it to your <code>Cargo.toml</code>:</p> <!>',1),L=p("<!> <p>You can simply install 🤗 Tokenizers with npm using:</p> <!>",1),S=p("<p></p> <!> <!> <!> <!> <p></p>",1);function O(z,J){P(J,!1),R(()=>{new URLSearchParams(window.location.search).get("fw")}),I();var g=S();C("t06thp",t=>{var l=Y();H(l,"content",U),h(t,l)});var u=a(d(g),2);G(u,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var m=a(u,2);r(m,{title:"Installation",local:"installation",headingTag:"h1"});var f=a(m,2);x(f,{python:!0,rust:!0,node:!0,$$slots:{python:(t,l)=>{c(t,{children:(s,w)=>{var e=B(),o=a(d(e),4);r(o,{title:"Installation with pip",local:"installation-with-pip",headingTag:"h2"});var i=a(o,4);n(i,{code:"cGlwJTIwaW5zdGFsbCUyMHRva2VuaXplcnM=",highlighted:"pip install tokenizers",lang:"bash",wrap:!1});var v=a(i,2);r(v,{title:"Installation from sources",local:"installation-from-sources",headingTag:"h2"});var b=a(v,6);n(b,{code:"Y3VybCUyMC0tcHJvdG8lMjAnJTNEaHR0cHMnJTIwLS10bHN2MS4yJTIwLXNTZiUyMGh0dHBzJTNBJTJGJTJGc2gucnVzdHVwLnJzJTIwJTdDJTIwc2g=",highlighted:'curl --proto <span class="hljs-string">&#x27;=https&#x27;</span> --tlsv1.2 -sSf https://sh.rustup.rs | sh',lang:"bash",wrap:!1});var y=a(b,4);n(y,{code:"cnVzdHVwJTIwdXBkYXRl",highlighted:"rustup update",lang:"bash",wrap:!1});var T=a(y,4);n(T,{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRnRva2VuaXplcnM=",highlighted:'git <span class="hljs-built_in">clone</span> https://github.com/huggingface/tokenizers',lang:"bash",wrap:!1});var _=a(T,4);n(_,{code:"Y2QlMjB0b2tlbml6ZXJzJTJGYmluZGluZ3MlMkZweXRob24=",highlighted:'<span class="hljs-built_in">cd</span> tokenizers/bindings/python',lang:"bash",wrap:!1});var k=a(_,4);n(k,{code:"cGlwJTIwaW5zdGFsbCUyMC1lJTIwLg==",highlighted:"pip install -e .",lang:"bash",wrap:!1}),h(s,e)}})},rust:(t,l)=>{c(t,{children:(s,w)=>{var e=X(),o=d(e);r(o,{title:"Crates.io",local:"cratesio",headingTag:"h2"});var i=a(o,6);n(i,{code:"Y2FyZ28lMjBhZGQlMjB0b2tlbml6ZXJz",highlighted:"cargo add tokenizers",lang:"bash",wrap:!1}),h(s,e)}})},node:(t,l)=>{c(t,{children:(s,w)=>{var e=L(),o=d(e);r(o,{title:"Installation with npm",local:"installation-with-npm",headingTag:"h2"});var i=a(o,4);n(i,{code:"bnBtJTIwaW5zdGFsbCUyMHRva2VuaXplcnM=",highlighted:"npm install tokenizers",lang:"bash",wrap:!1}),h(s,e)}})}}});var M=a(f,2);$(M,{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/installation.mdx"}),j(2),h(z,g),Z()}export{O as component};