tokenizers/pr_2139/en/_app/immutable/nodes/13.85cc6cfc.js

import{s as q,o as G,n as R}from"../chunks/scheduler.7c59faff.js";import{S as U,i as W,e as E,s as _,c as z,h as J,a as T,d as s,b as h,f as D,g as k,j as H,k as P,l as u,m as b,t as v,n as x,o as y,p as N}from"../chunks/index.09bb5655.js";import{C as K,H as j,E as Q}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.fb0d6fe2.js";import{D as S}from"../chunks/Docstring.1c25d4eb.js";import{T as X,M as B}from"../chunks/TokenizersLanguageContent.0fc17a7a.js";function Y(f){let e,a,t,o,l,$,g,c,m,C,p,L="Build an EncodingVisualizer",V,d,n,i,A,F="Build a visualization of the given text",I;return e=new j({props:{title:"Annotation",local:"tokenizers.tools.Annotation",headingTag:"h2"}}),o=new S({props:{name:"class tokenizers.tools.Annotation",anchor:"tokenizers.tools.Annotation",parameters:[{name:"start",val:": int"},{name:"end",val:": int"},{name:"label",val:": str"}],source:"https://github.com/huggingface/tokenizers/blob/vr_2139/bindings/python/py_src/tokenizers/tools/visualizer.py#L16"}}),$=new j({props:{title:"EncodingVisualizer",local:"tokenizers.tools.EncodingVisualizer",headingTag:"h2"}}),m=new S({props:{name:"class tokenizers.tools.EncodingVisualizer",anchor:"tokenizers.tools.EncodingVisualizer",parameters:[{name:"tokenizer",val:": Tokenizer"},{name:"default_to_notebook",val:": bool = True"},{name:"annotation_converter",val:": typing.Optional[typing.Callable[[typing.Any], tokenizers.tools.visualizer.Annotation]] = None"}],parametersDescription:[{anchor:"tokenizers.tools.EncodingVisualizer.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/tokenizers/pr_2139/en/api/tokenizer#tokenizers.Tokenizer">Tokenizer</a>) &#x2014;
A tokenizer instance`,name:"tokenizer"},{anchor:"tokenizers.tools.EncodingVisualizer.default_to_notebook",description:`<strong>default_to_notebook</strong> (<code>bool</code>) &#x2014;
Whether to render html output in a notebook by default`,name:"default_to_notebook"},{anchor:"tokenizers.tools.EncodingVisualizer.annotation_converter",description:`<strong>annotation_converter</strong> (<code>Callable</code>, <em>optional</em>) &#x2014;
An optional (lambda) function that takes an annotation in any format and returns
an Annotation object`,name:"annotation_converter"}],source:"https://github.com/huggingface/tokenizers/blob/vr_2139/bindings/python/py_src/tokenizers/tools/visualizer.py#L67"}}),n=new S({props:{name:"__call__",anchor:"tokenizers.tools.EncodingVisualizer.__call__",parameters:[{name:"text",val:": str"},{name:"annotations",val:": typing.Optional[typing.List[typing.Any]] = None"},{name:"default_to_notebook",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"tokenizers.tools.EncodingVisualizer.__call__.text",description:`<strong>text</strong> (<code>str</code>) &#x2014;
The text to tokenize`,name:"text"},{anchor:"tokenizers.tools.EncodingVisualizer.__call__.annotations",description:`<strong>annotations</strong> (<code>List[Annotation]</code>, <em>optional</em>) &#x2014;
An optional list of annotations of the text. The can either be an annotation class
or anything else if you instantiated the visualizer with a converter function`,name:"annotations"},{anchor:"tokenizers.tools.EncodingVisualizer.__call__.default_to_notebook",description:`<strong>default_to_notebook</strong> (<code>bool</code>, <em>optional</em>, defaults to <em>False</em>) &#x2014;
If True, will render the html in a notebook. Otherwise returns an html string.`,name:"default_to_notebook"}],source:"https://github.com/huggingface/tokenizers/blob/vr_2139/bindings/python/py_src/tokenizers/tools/visualizer.py#L110",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The HTML string if default_to_notebook is False, otherwise (default) returns None and
renders the HTML in the notebook</p>
`}}),{c(){z(e.$$.fragment),a=_(),t=E("div"),z(o.$$.fragment),l=_(),z($.$$.fragment),g=_(),c=E("div"),z(m.$$.fragment),C=_(),p=E("p"),p.textContent=L,V=_(),d=E("div"),z(n.$$.fragment),i=_(),A=E("p"),A.textContent=F,this.h()},l(r){k(e.$$.fragment,r),a=h(r),t=T(r,"DIV",{class:!0});var w=D(t);k(o.$$.fragment,w),w.forEach(s),l=h(r),k($.$$.fragment,r),g=h(r),c=T(r,"DIV",{class:!0});var M=D(c);k(m.$$.fragment,M),C=h(M),p=T(M,"P",{"data-svelte-h":!0}),N(p)!=="svelte-jh81sk"&&(p.textContent=L),V=h(M),d=T(M,"DIV",{class:!0});var O=D(d);k(n.$$.fragment,O),i=h(O),A=T(O,"P",{"data-svelte-h":!0}),N(A)!=="svelte-i1ht2h"&&(A.textContent=F),O.forEach(s),M.forEach(s),this.h()},h(){H(t,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),H(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),H(c,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(r,w){b(e,r,w),u(r,a,w),u(r,t,w),b(o,t,null),u(r,l,w),b($,r,w),u(r,g,w),u(r,c,w),b(m,c,null),P(c,C),P(c,p),P(c,V),P(c,d),b(n,d,null),P(d,i),P(d,A),I=!0},p:R,i(r){I||(v(e.$$.fragment,r),v(o.$$.fragment,r),v($.$$.fragment,r),v(m.$$.fragment,r),v(n.$$.fragment,r),I=!0)},o(r){x(e.$$.fragment,r),x(o.$$.fragment,r),x($.$$.fragment,r),x(m.$$.fragment,r),x(n.$$.fragment,r),I=!1},d(r){r&&(s(a),s(t),s(l),s(g),s(c)),y(e,r),y(o),y($,r),y(m),y(n)}}}function Z(f){let e,a;return e=new B({props:{$$slots:{default:[Y]},$$scope:{ctx:f}}}),{c(){z(e.$$.fragment)},l(t){k(e.$$.fragment,t)},m(t,o){b(e,t,o),a=!0},p(t,o){const l={};o&2&&(l.$$scope={dirty:o,ctx:t}),e.$set(l)},i(t){a||(v(e.$$.fragment,t),a=!0)},o(t){x(e.$$.fragment,t),a=!1},d(t){y(e,t)}}}function tt(f){let e,a='The Rust API Reference is available directly on the <a href="https://docs.rs/tokenizers/latest/tokenizers/" rel="nofollow">Docs.rs</a> website.';return{c(){e=E("p"),e.innerHTML=a},l(t){e=T(t,"P",{"data-svelte-h":!0}),N(e)!=="svelte-4ytcyb"&&(e.innerHTML=a)},m(t,o){u(t,e,o)},p:R,d(t){t&&s(e)}}}function et(f){let e,a;return e=new B({props:{$$slots:{default:[tt]},$$scope:{ctx:f}}}),{c(){z(e.$$.fragment)},l(t){k(e.$$.fragment,t)},m(t,o){b(e,t,o),a=!0},p(t,o){const l={};o&2&&(l.$$scope={dirty:o,ctx:t}),e.$set(l)},i(t){a||(v(e.$$.fragment,t),a=!0)},o(t){x(e.$$.fragment,t),a=!1},d(t){y(e,t)}}}function nt(f){let e,a="The node API has not been documented yet.";return{c(){e=E("p"),e.textContent=a},l(t){e=T(t,"P",{"data-svelte-h":!0}),N(e)!=="svelte-1mrchm6"&&(e.textContent=a)},m(t,o){u(t,e,o)},p:R,d(t){t&&s(e)}}}function ot(f){let e,a;return e=new B({props:{$$slots:{default:[nt]},$$scope:{ctx:f}}}),{c(){z(e.$$.fragment)},l(t){k(e.$$.fragment,t)},m(t,o){b(e,t,o),a=!0},p(t,o){const l={};o&2&&(l.$$scope={dirty:o,ctx:t}),e.$set(l)},i(t){a||(v(e.$$.fragment,t),a=!0)},o(t){x(e.$$.fragment,t),a=!1},d(t){y(e,t)}}}function rt(f){let e,a,t,o,l,$,g,c,m,C,p,L,V,d;return l=new K({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),g=new j({props:{title:"Visualizer",local:"visualizer",headingTag:"h1"}}),m=new X({props:{python:!0,rust:!0,node:!0,$$slots:{node:[ot],rust:[et],python:[Z]},$$scope:{ctx:f}}}),p=new Q({props:{source:"https://github.com/huggingface/tokenizers/blob/main/docs/source-doc-builder/api/visualizer.mdx"}}),{c(){e=E("meta"),a=_(),t=E("p"),o=_(),z(l.$$.fragment),$=_(),z(g.$$.fragment),c=_(),z(m.$$.fragment),C=_(),z(p.$$.fragment),L=_(),V=E("p"),this.h()},l(n){const i=J("svelte-u9bgzb",document.head);e=T(i,"META",{name:!0,content:!0}),i.forEach(s),a=h(n),t=T(n,"P",{}),D(t).forEach(s),o=h(n),k(l.$$.fragment,n),$=h(n),k(g.$$.fragment,n),c=h(n),k(m.$$.fragment,n),C=h(n),k(p.$$.fragment,n),L=h(n),V=T(n,"P",{}),D(V).forEach(s),this.h()},h(){H(e,"name","hf:doc:metadata"),H(e,"content",at)},m(n,i){P(document.head,e),u(n,a,i),u(n,t,i),u(n,o,i),b(l,n,i),u(n,$,i),b(g,n,i),u(n,c,i),b(m,n,i),u(n,C,i),b(p,n,i),u(n,L,i),u(n,V,i),d=!0},p(n,[i]){const A={};i&2&&(A.$$scope={dirty:i,ctx:n}),m.$set(A)},i(n){d||(v(l.$$.fragment,n),v(g.$$.fragment,n),v(m.$$.fragment,n),v(p.$$.fragment,n),d=!0)},o(n){x(l.$$.fragment,n),x(g.$$.fragment,n),x(m.$$.fragment,n),x(p.$$.fragment,n),d=!1},d(n){n&&(s(a),s(t),s(o),s($),s(c),s(C),s(L),s(V)),s(e),y(l,n),y(g,n),y(m,n),y(p,n)}}}const at='{"title":"Visualizer","local":"visualizer","sections":[{"title":"Annotation","local":"tokenizers.tools.Annotation","sections":[],"depth":2},{"title":"EncodingVisualizer","local":"tokenizers.tools.EncodingVisualizer","sections":[],"depth":2}],"depth":1}';function it(f){return G(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pt extends U{constructor(e){super(),W(this,e,it,rt,q,{})}}export{pt as component};