Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / transformers.js /pr_1113 /en /api /utils /data-structures.html

rtrm

29 days ago

download

raw

64.8 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"utils/data-structures","local":"utilsdata-structures","sections":[{"title":"utils/data-structures.PriorityQueue","local":"utilsdata-structurespriorityqueue","sections":[{"title":"new PriorityQueue(comparator)","local":"new-priorityqueuecomparator","sections":[],"depth":3},{"title":"priorityQueue.size","local":"priorityqueuesize","sections":[],"depth":3},{"title":"priorityQueue.isEmpty() ⇒ <code> boolean </code>","local":"priorityqueueisempty--code-boolean-code","sections":[],"depth":3},{"title":"priorityQueue.peek() ⇒ <code> any </code>","local":"priorityqueuepeek--code-any-code","sections":[],"depth":3},{"title":"priorityQueue.push(...values) ⇒ <code> number </code>","local":"priorityqueuepushvalues--code-number-code","sections":[],"depth":3},{"title":"priorityQueue.extend(values) ⇒ <code> number </code>","local":"priorityqueueextendvalues--code-number-code","sections":[],"depth":3},{"title":"priorityQueue.pop() ⇒ <code> any </code>","local":"priorityqueuepop--code-any-code","sections":[],"depth":3},{"title":"priorityQueue.replace(value) ⇒ <code> * </code>","local":"priorityqueuereplacevalue--code--code","sections":[],"depth":3},{"title":"priorityQueue._siftUpFrom(node)","local":"priorityqueuesiftupfromnode","sections":[],"depth":3}],"depth":2},{"title":"utils/data-structures.CharTrie","local":"utilsdata-structureschartrie","sections":[{"title":"charTrie.extend(texts)","local":"chartrieextendtexts","sections":[],"depth":3},{"title":"charTrie.push(text)","local":"chartriepushtext","sections":[],"depth":3},{"title":"charTrie.commonPrefixSearch(text)","local":"chartriecommonprefixsearchtext","sections":[],"depth":3}],"depth":2},{"title":"utils/data-structures.TokenLattice","local":"utilsdata-structurestokenlattice","sections":[{"title":"new TokenLattice(sentence, bosTokenId, eosTokenId)","local":"new-tokenlatticesentence-bostokenid-eostokenid","sections":[],"depth":3},{"title":"tokenLattice.insert(pos, length, score, tokenId)","local":"tokenlatticeinsertpos-length-score-tokenid","sections":[],"depth":3},{"title":"tokenLattice.viterbi() ⇒ <code> Array. < TokenLatticeNode > </code>","local":"tokenlatticeviterbi--code-array--tokenlatticenode--code","sections":[],"depth":3},{"title":"tokenLattice.piece(node) ⇒ <code> string </code>","local":"tokenlatticepiecenode--code-string-code","sections":[],"depth":3},{"title":"tokenLattice.tokens() ⇒ <code> Array. < string > </code>","local":"tokenlatticetokens--code-array--string--code","sections":[],"depth":3},{"title":"tokenLattice.tokenIds() ⇒ <code> Array. < number > </code>","local":"tokenlatticetokenids--code-array--number--code","sections":[],"depth":3}],"depth":2},{"title":"utils/data-structures~CharTrieNode","local":"utilsdata-structureschartrienode","sections":[{"title":"new CharTrieNode(isLeaf, children)","local":"new-chartrienodeisleaf-children","sections":[],"depth":3},{"title":"CharTrieNode.default() ⇒ <code> CharTrieNode </code>","local":"chartrienodedefault--code-chartrienode-code","sections":[],"depth":3}],"depth":2},{"title":"utils/data-structures~TokenLatticeNode","local":"utilsdata-structurestokenlatticenode","sections":[{"title":"new TokenLatticeNode(tokenId, nodeId, pos, length, score)","local":"new-tokenlatticenodetokenid-nodeid-pos-length-score","sections":[],"depth":3},{"title":"tokenLatticeNode.clone() ⇒ <code> TokenLatticeNode </code>","local":"tokenlatticenodeclone--code-tokenlatticenode-code","sections":[],"depth":3}],"depth":2}],"depth":1}">
	<link href="/docs/transformers.js/pr_1113/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/entry/start.88a6e140.js">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/chunks/scheduler.0219f8bd.js">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/chunks/singletons.c59c6d8d.js">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/chunks/paths.8e090985.js">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/entry/app.0003020d.js">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/chunks/index.f61edf3b.js">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/nodes/0.25c65ab2.js">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/nodes/18.57eec696.js">
	<link rel="modulepreload" href="/docs/transformers.js/pr_1113/en/_app/immutable/chunks/EditOnGithub.48fa589f.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"utils/data-structures","local":"utilsdata-structures","sections":[{"title":"utils/data-structures.PriorityQueue","local":"utilsdata-structurespriorityqueue","sections":[{"title":"new PriorityQueue(comparator)","local":"new-priorityqueuecomparator","sections":[],"depth":3},{"title":"priorityQueue.size","local":"priorityqueuesize","sections":[],"depth":3},{"title":"priorityQueue.isEmpty() ⇒ <code> boolean </code>","local":"priorityqueueisempty--code-boolean-code","sections":[],"depth":3},{"title":"priorityQueue.peek() ⇒ <code> any </code>","local":"priorityqueuepeek--code-any-code","sections":[],"depth":3},{"title":"priorityQueue.push(...values) ⇒ <code> number </code>","local":"priorityqueuepushvalues--code-number-code","sections":[],"depth":3},{"title":"priorityQueue.extend(values) ⇒ <code> number </code>","local":"priorityqueueextendvalues--code-number-code","sections":[],"depth":3},{"title":"priorityQueue.pop() ⇒ <code> any </code>","local":"priorityqueuepop--code-any-code","sections":[],"depth":3},{"title":"priorityQueue.replace(value) ⇒ <code> * </code>","local":"priorityqueuereplacevalue--code--code","sections":[],"depth":3},{"title":"priorityQueue._siftUpFrom(node)","local":"priorityqueuesiftupfromnode","sections":[],"depth":3}],"depth":2},{"title":"utils/data-structures.CharTrie","local":"utilsdata-structureschartrie","sections":[{"title":"charTrie.extend(texts)","local":"chartrieextendtexts","sections":[],"depth":3},{"title":"charTrie.push(text)","local":"chartriepushtext","sections":[],"depth":3},{"title":"charTrie.commonPrefixSearch(text)","local":"chartriecommonprefixsearchtext","sections":[],"depth":3}],"depth":2},{"title":"utils/data-structures.TokenLattice","local":"utilsdata-structurestokenlattice","sections":[{"title":"new TokenLattice(sentence, bosTokenId, eosTokenId)","local":"new-tokenlatticesentence-bostokenid-eostokenid","sections":[],"depth":3},{"title":"tokenLattice.insert(pos, length, score, tokenId)","local":"tokenlatticeinsertpos-length-score-tokenid","sections":[],"depth":3},{"title":"tokenLattice.viterbi() ⇒ <code> Array. < TokenLatticeNode > </code>","local":"tokenlatticeviterbi--code-array--tokenlatticenode--code","sections":[],"depth":3},{"title":"tokenLattice.piece(node) ⇒ <code> string </code>","local":"tokenlatticepiecenode--code-string-code","sections":[],"depth":3},{"title":"tokenLattice.tokens() ⇒ <code> Array. < string > </code>","local":"tokenlatticetokens--code-array--string--code","sections":[],"depth":3},{"title":"tokenLattice.tokenIds() ⇒ <code> Array. < number > </code>","local":"tokenlatticetokenids--code-array--number--code","sections":[],"depth":3}],"depth":2},{"title":"utils/data-structures~CharTrieNode","local":"utilsdata-structureschartrienode","sections":[{"title":"new CharTrieNode(isLeaf, children)","local":"new-chartrienodeisleaf-children","sections":[],"depth":3},{"title":"CharTrieNode.default() ⇒ <code> CharTrieNode </code>","local":"chartrienodedefault--code-chartrienode-code","sections":[],"depth":3}],"depth":2},{"title":"utils/data-structures~TokenLatticeNode","local":"utilsdata-structurestokenlatticenode","sections":[{"title":"new TokenLatticeNode(tokenId, nodeId, pos, length, score)","local":"new-tokenlatticenodetokenid-nodeid-pos-length-score","sections":[],"depth":3},{"title":"tokenLatticeNode.clone() ⇒ <code> TokenLatticeNode </code>","local":"tokenlatticenodeclone--code-tokenlatticenode-code","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <a id="module_utils/data-structures" class="group"></a> <h1 class="relative group"><a id="utilsdata-structures" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structures"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures</span></h1> <p data-svelte-h="svelte-1aeeeid">Custom data structures.</p> <p data-svelte-h="svelte-1r3ntfo">These are only used internally, meaning an end-user shouldn’t
	need to access anything here.</p> <ul data-svelte-h="svelte-kea2e6"><li><a href="#module_utils/data-structures">utils/data-structures</a><ul><li><em>static</em><ul><li><a href="#module_utils/data-structures.PriorityQueue">.PriorityQueue</a><ul><li><a href="#new_module_utils/data-structures.PriorityQueue_new"><code>new PriorityQueue(comparator)</code></a></li> <li><a href="#module_utils/data-structures.PriorityQueue+size"><code>.size</code></a></li> <li><a href="#module_utils/data-structures.PriorityQueue+isEmpty"><code>.isEmpty()</code></a> ⇒ <code>boolean</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+peek"><code>.peek()</code></a> ⇒ <code>any</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+push"><code>.push(...values)</code></a> ⇒ <code>number</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+extend"><code>.extend(values)</code></a> ⇒ <code>number</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+pop"><code>.pop()</code></a> ⇒ <code>any</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+replace"><code>.replace(value)</code></a> ⇒ <code>*</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+_siftUpFrom"><code>._siftUpFrom(node)</code></a></li></ul></li> <li><a href="#module_utils/data-structures.CharTrie">.CharTrie</a><ul><li><a href="#module_utils/data-structures.CharTrie+extend"><code>.extend(texts)</code></a></li> <li><a href="#module_utils/data-structures.CharTrie+push"><code>.push(text)</code></a></li> <li><a href="#module_utils/data-structures.CharTrie+commonPrefixSearch"><code>.commonPrefixSearch(text)</code></a></li></ul></li> <li><a href="#module_utils/data-structures.TokenLattice">.TokenLattice</a><ul><li><a href="#new_module_utils/data-structures.TokenLattice_new"><code>new TokenLattice(sentence, bosTokenId, eosTokenId)</code></a></li> <li><a href="#module_utils/data-structures.TokenLattice+insert"><code>.insert(pos, length, score, tokenId)</code></a></li> <li><a href="#module_utils/data-structures.TokenLattice+viterbi"><code>.viterbi()</code></a> ⇒ <code>Array.<TokenLatticeNode></code></li> <li><a href="#module_utils/data-structures.TokenLattice+piece"><code>.piece(node)</code></a> ⇒ <code>string</code></li> <li><a href="#module_utils/data-structures.TokenLattice+tokens"><code>.tokens()</code></a> ⇒ <code>Array.<string></code></li> <li><a href="#module_utils/data-structures.TokenLattice+tokenIds"><code>.tokenIds()</code></a> ⇒ <code>Array.<number></code></li></ul></li></ul></li> <li><em>inner</em><ul><li><a href="#module_utils/data-structures..CharTrieNode">~CharTrieNode</a><ul><li><a href="#new_module_utils/data-structures..CharTrieNode_new"><code>new CharTrieNode(isLeaf, children)</code></a></li> <li><a href="#module_utils/data-structures..CharTrieNode.default"><code>.default()</code></a> ⇒ <code>CharTrieNode</code></li></ul></li> <li><a href="#module_utils/data-structures..TokenLatticeNode">~TokenLatticeNode</a><ul><li><a href="#new_module_utils/data-structures..TokenLatticeNode_new"><code>new TokenLatticeNode(tokenId, nodeId, pos, length, score)</code></a></li> <li><a href="#module_utils/data-structures..TokenLatticeNode+clone"><code>.clone()</code></a> ⇒ <code>TokenLatticeNode</code></li></ul></li></ul></li></ul></li></ul> <hr> <a id="module_utils/data-structures.PriorityQueue" class="group"></a> <h2 class="relative group"><a id="utilsdata-structurespriorityqueue" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structurespriorityqueue"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures.PriorityQueue</span></h2> <p data-svelte-h="svelte-1ux83uj">Efficient Heap-based Implementation of a Priority Queue.
	It uses an array-based binary heap, where the root is at index <code>0</code>, and the
	children of node <code>i</code> are located at indices <code>2i + 1</code> and <code>2i + 2</code>, respectively.</p> <p data-svelte-h="svelte-vi2pz9">Adapted from the following sources:</p> <ul data-svelte-h="svelte-qlvgg9"><li><a href="https://stackoverflow.com/a/42919752/13989043" rel="nofollow">https://stackoverflow.com/a/42919752/13989043</a> (original)</li> <li><a href="https://github.com/belladoreai/llama-tokenizer-js" rel="nofollow">https://github.com/belladoreai/llama-tokenizer-js</a> (minor improvements)</li></ul> <p data-svelte-h="svelte-1saulos"><strong>Kind</strong>: static class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-fzyqbp"><li><a href="#module_utils/data-structures.PriorityQueue">.PriorityQueue</a><ul><li><a href="#new_module_utils/data-structures.PriorityQueue_new"><code>new PriorityQueue(comparator)</code></a></li> <li><a href="#module_utils/data-structures.PriorityQueue+size"><code>.size</code></a></li> <li><a href="#module_utils/data-structures.PriorityQueue+isEmpty"><code>.isEmpty()</code></a> ⇒ <code>boolean</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+peek"><code>.peek()</code></a> ⇒ <code>any</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+push"><code>.push(...values)</code></a> ⇒ <code>number</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+extend"><code>.extend(values)</code></a> ⇒ <code>number</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+pop"><code>.pop()</code></a> ⇒ <code>any</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+replace"><code>.replace(value)</code></a> ⇒ <code></code></li> <li><a href="#module_utils/data-structures.PriorityQueue+_siftUpFrom"><code>._siftUpFrom(node)</code></a></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures.PriorityQueue_new" class="group"></a> <h3 class="relative group"><a id="new-priorityqueuecomparator" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-priorityqueuecomparator"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new PriorityQueue(comparator)</span></h3> <p data-svelte-h="svelte-1dgukb8">Create a new PriorityQueue.</p> <table data-svelte-h="svelte-15lsmj7"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>comparator</td><td><code>function</code></td><td><p>Comparator function to determine priority. Defaults to a MaxHeap.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.PriorityQueue+size" class="group"></a> <h3 class="relative group"><a id="priorityqueuesize" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuesize"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.size</span></h3> <p data-svelte-h="svelte-19ll7y9">The size of the queue</p> <p data-svelte-h="svelte-1spompf"><strong>Kind</strong>: instance property of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a></p> <hr> <a id="module_utils/data-structures.PriorityQueue+isEmpty" class="group"></a> <h3 class="relative group"><a id="priorityqueueisempty--code-boolean-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueueisempty--code-boolean-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.isEmpty() ⇒ <code> boolean </code></span></h3> <p data-svelte-h="svelte-iwo0ja">Check if the queue is empty.</p> <p data-svelte-h="svelte-usw3zo"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>boolean</code> - <code>true</code> if the queue is empty, <code>false</code> otherwise.</p> <hr> <a id="module_utils/data-structures.PriorityQueue+peek" class="group"></a> <h3 class="relative group"><a id="priorityqueuepeek--code-any-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuepeek--code-any-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.peek() ⇒ <code> any </code></span></h3> <p data-svelte-h="svelte-1g38rzl">Return the element with the highest priority in the queue.</p> <p data-svelte-h="svelte-1jn2os3"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>any</code> - The highest priority element in the queue.</p> <hr> <a id="module_utils/data-structures.PriorityQueue+push" class="group"></a> <h3 class="relative group"><a id="priorityqueuepushvalues--code-number-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuepushvalues--code-number-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.push(...values) ⇒ <code> number </code></span></h3> <p data-svelte-h="svelte-1o3p42d">Add one or more elements to the queue.</p> <p data-svelte-h="svelte-1hty3q7"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>number</code> - The new size of the queue.</p> <table data-svelte-h="svelte-19plha1"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>...values</td><td><code>any</code></td><td><p>The values to push into the queue.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.PriorityQueue+extend" class="group"></a> <h3 class="relative group"><a id="priorityqueueextendvalues--code-number-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueueextendvalues--code-number-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.extend(values) ⇒ <code> number </code></span></h3> <p data-svelte-h="svelte-1rf8gmz">Add multiple elements to the queue.</p> <p data-svelte-h="svelte-1hty3q7"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>number</code> - The new size of the queue.</p> <table data-svelte-h="svelte-txh99d"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>values</td><td><code>Array.<any></code></td><td><p>The values to push into the queue.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.PriorityQueue+pop" class="group"></a> <h3 class="relative group"><a id="priorityqueuepop--code-any-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuepop--code-any-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.pop() ⇒ <code> any </code></span></h3> <p data-svelte-h="svelte-1l3oksc">Remove and return the element with the highest priority in the queue.</p> <p data-svelte-h="svelte-vpswb8"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>any</code> - The element with the highest priority in the queue.</p> <hr> <a id="module_utils/data-structures.PriorityQueue+replace" class="group"></a> <h3 class="relative group"><a id="priorityqueuereplacevalue--code--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuereplacevalue--code--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.replace(value) ⇒ <code> </code></span></h3> <p data-svelte-h="svelte-7s4t0d">Replace the element with the highest priority in the queue with a new value.</p> <p data-svelte-h="svelte-ckytvb"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code></code> - The replaced value.</p> <table data-svelte-h="svelte-1xv2m9q"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>value</td><td><code></code></td><td><p>The new value.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.PriorityQueue+_siftUpFrom" class="group"></a> <h3 class="relative group"><a id="priorityqueuesiftupfromnode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuesiftupfromnode"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue._siftUpFrom(node)</span></h3> <p data-svelte-h="svelte-hj5sse">Helper function to sift up from a given node.</p> <p data-svelte-h="svelte-6p6lmz"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a></p> <table data-svelte-h="svelte-fj2fu5"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>node</td><td><code>number</code></td><td><p>The index of the node to start sifting up from.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.CharTrie" class="group"></a> <h2 class="relative group"><a id="utilsdata-structureschartrie" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structureschartrie"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures.CharTrie</span></h2> <p data-svelte-h="svelte-nklgrc">A trie structure to efficiently store and search for strings.</p> <p data-svelte-h="svelte-1saulos"><strong>Kind</strong>: static class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-1yt2jf0"><li><a href="#module_utils/data-structures.CharTrie">.CharTrie</a><ul><li><a href="#module_utils/data-structures.CharTrie+extend"><code>.extend(texts)</code></a></li> <li><a href="#module_utils/data-structures.CharTrie+push"><code>.push(text)</code></a></li> <li><a href="#module_utils/data-structures.CharTrie+commonPrefixSearch"><code>.commonPrefixSearch(text)</code></a></li></ul></li></ul> <hr> <a id="module_utils/data-structures.CharTrie+extend" class="group"></a> <h3 class="relative group"><a id="chartrieextendtexts" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#chartrieextendtexts"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>charTrie.extend(texts)</span></h3> <p data-svelte-h="svelte-moeq8l">Adds one or more <code>texts</code> to the trie.</p> <p data-svelte-h="svelte-33rax3"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.CharTrie"><code>CharTrie</code></a></p> <table data-svelte-h="svelte-18dtsnv"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>texts</td><td><code>Array.<string></code></td><td><p>The strings to add to the trie.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.CharTrie+push" class="group"></a> <h3 class="relative group"><a id="chartriepushtext" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#chartriepushtext"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>charTrie.push(text)</span></h3> <p data-svelte-h="svelte-4q846v">Adds text to the trie.</p> <p data-svelte-h="svelte-33rax3"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.CharTrie"><code>CharTrie</code></a></p> <table data-svelte-h="svelte-hqwgb9"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>text</td><td><code>string</code></td><td><p>The string to add to the trie.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.CharTrie+commonPrefixSearch" class="group"></a> <h3 class="relative group"><a id="chartriecommonprefixsearchtext" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#chartriecommonprefixsearchtext"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>charTrie.commonPrefixSearch(text)</span></h3> <p data-svelte-h="svelte-bmaw98">Searches the trie for all strings with a common prefix of <code>text</code>.</p> <p data-svelte-h="svelte-33rax3"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.CharTrie"><code>CharTrie</code></a></p> <table data-svelte-h="svelte-18qxh0x"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>text</td><td><code>string</code></td><td><p>The common prefix to search for.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.TokenLattice" class="group"></a> <h2 class="relative group"><a id="utilsdata-structurestokenlattice" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structurestokenlattice"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures.TokenLattice</span></h2> <p data-svelte-h="svelte-1xkinx1">A lattice data structure to be used for tokenization.</p> <p data-svelte-h="svelte-1saulos"><strong>Kind</strong>: static class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-12y4jvx"><li><a href="#module_utils/data-structures.TokenLattice">.TokenLattice</a><ul><li><a href="#new_module_utils/data-structures.TokenLattice_new"><code>new TokenLattice(sentence, bosTokenId, eosTokenId)</code></a></li> <li><a href="#module_utils/data-structures.TokenLattice+insert"><code>.insert(pos, length, score, tokenId)</code></a></li> <li><a href="#module_utils/data-structures.TokenLattice+viterbi"><code>.viterbi()</code></a> ⇒ <code>Array.<TokenLatticeNode></code></li> <li><a href="#module_utils/data-structures.TokenLattice+piece"><code>.piece(node)</code></a> ⇒ <code>string</code></li> <li><a href="#module_utils/data-structures.TokenLattice+tokens"><code>.tokens()</code></a> ⇒ <code>Array.<string></code></li> <li><a href="#module_utils/data-structures.TokenLattice+tokenIds"><code>.tokenIds()</code></a> ⇒ <code>Array.<number></code></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures.TokenLattice_new" class="group"></a> <h3 class="relative group"><a id="new-tokenlatticesentence-bostokenid-eostokenid" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-tokenlatticesentence-bostokenid-eostokenid"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new TokenLattice(sentence, bosTokenId, eosTokenId)</span></h3> <p data-svelte-h="svelte-ziiz62">Creates a new TokenLattice instance.</p> <table data-svelte-h="svelte-1h8dv5n"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>sentence</td><td><code>string</code></td><td><p>The input sentence to be tokenized.</p></td> </tr><tr><td>bosTokenId</td><td><code>number</code></td><td><p>The beginning-of-sequence token ID.</p></td> </tr><tr><td>eosTokenId</td><td><code>number</code></td><td><p>The end-of-sequence token ID.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.TokenLattice+insert" class="group"></a> <h3 class="relative group"><a id="tokenlatticeinsertpos-length-score-tokenid" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticeinsertpos-length-score-tokenid"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.insert(pos, length, score, tokenId)</span></h3> <p data-svelte-h="svelte-1hzq3te">Inserts a new token node into the token lattice.</p> <p data-svelte-h="svelte-1bjyh7n"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a></p> <table data-svelte-h="svelte-1a01i2a"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>pos</td><td><code>number</code></td><td><p>The starting position of the token.</p></td> </tr><tr><td>length</td><td><code>number</code></td><td><p>The length of the token.</p></td> </tr><tr><td>score</td><td><code>number</code></td><td><p>The score of the token.</p></td> </tr><tr><td>tokenId</td><td><code>number</code></td><td><p>The token ID of the token.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.TokenLattice+viterbi" class="group"></a> <h3 class="relative group"><a id="tokenlatticeviterbi--code-array--tokenlatticenode--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticeviterbi--code-array--tokenlatticenode--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.viterbi() ⇒ <code> Array. < TokenLatticeNode > </code></span></h3> <p data-svelte-h="svelte-jv0r">Implements the Viterbi algorithm to compute the most likely sequence of tokens.</p> <p data-svelte-h="svelte-1hp113b"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a><br> <strong>Returns</strong>: <code>Array.<TokenLatticeNode></code> - The most likely sequence of tokens.</p> <hr> <a id="module_utils/data-structures.TokenLattice+piece" class="group"></a> <h3 class="relative group"><a id="tokenlatticepiecenode--code-string-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticepiecenode--code-string-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.piece(node) ⇒ <code> string </code></span></h3> <p data-svelte-h="svelte-1n9zjoi"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a><br> <strong>Returns</strong>: <code>string</code> - The array of nodes representing the most likely sequence of tokens.</p> <table data-svelte-h="svelte-6nufyb"><thead><tr><th>Param</th><th>Type</th></tr></thead> <tbody><tr><td>node</td><td><code>TokenLatticeNode</code></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.TokenLattice+tokens" class="group"></a> <h3 class="relative group"><a id="tokenlatticetokens--code-array--string--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticetokens--code-array--string--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.tokens() ⇒ <code> Array. < string > </code></span></h3> <p data-svelte-h="svelte-1ipxykz"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a><br> <strong>Returns</strong>: <code>Array.<string></code> - The most likely sequence of tokens.</p> <hr> <a id="module_utils/data-structures.TokenLattice+tokenIds" class="group"></a> <h3 class="relative group"><a id="tokenlatticetokenids--code-array--number--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticetokenids--code-array--number--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.tokenIds() ⇒ <code> Array. < number > </code></span></h3> <p data-svelte-h="svelte-1nolezc"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a><br> <strong>Returns</strong>: <code>Array.<number></code> - The most likely sequence of token ids.</p> <hr> <a id="module_utils/data-structures..CharTrieNode" class="group"></a> <h2 class="relative group"><a id="utilsdata-structureschartrienode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structureschartrienode"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures~CharTrieNode</span></h2> <p data-svelte-h="svelte-1auz3rh">Represents a node in a character trie.</p> <p data-svelte-h="svelte-1p9ver0"><strong>Kind</strong>: inner class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-4v7sww"><li><a href="#module_utils/data-structures..CharTrieNode">~CharTrieNode</a><ul><li><a href="#new_module_utils/data-structures..CharTrieNode_new"><code>new CharTrieNode(isLeaf, children)</code></a></li> <li><a href="#module_utils/data-structures..CharTrieNode.default"><code>.default()</code></a> ⇒ <code>CharTrieNode</code></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures..CharTrieNode_new" class="group"></a> <h3 class="relative group"><a id="new-chartrienodeisleaf-children" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-chartrienodeisleaf-children"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new CharTrieNode(isLeaf, children)</span></h3> <p data-svelte-h="svelte-mt1jex">Create a new CharTrieNode.</p> <table data-svelte-h="svelte-nk427c"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>isLeaf</td><td><code>boolean</code></td><td><p>Whether the node is a leaf node or not.</p></td> </tr><tr><td>children</td><td><code>Map.<string, CharTrieNode></code></td><td><p>A map containing the node's children, where the key is a character and the value is a <code>CharTrieNode</code>.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures..CharTrieNode.default" class="group"></a> <h3 class="relative group"><a id="chartrienodedefault--code-chartrienode-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#chartrienodedefault--code-chartrienode-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>CharTrieNode.default() ⇒ <code> CharTrieNode </code></span></h3> <p data-svelte-h="svelte-9cjdkj">Returns a new <code>CharTrieNode</code> instance with default values.</p> <p data-svelte-h="svelte-3pigfy"><strong>Kind</strong>: static method of <a href="#module_utils/data-structures..CharTrieNode"><code>CharTrieNode</code></a><br> <strong>Returns</strong>: <code>CharTrieNode</code> - A new <code>CharTrieNode</code> instance with <code>isLeaf</code> set to <code>false</code> and an empty <code>children</code> map.</p> <hr> <a id="module_utils/data-structures..TokenLatticeNode" class="group"></a> <h2 class="relative group"><a id="utilsdata-structurestokenlatticenode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structurestokenlatticenode"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures~TokenLatticeNode</span></h2> <p data-svelte-h="svelte-1p9ver0"><strong>Kind</strong>: inner class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-r6qh2h"><li><a href="#module_utils/data-structures..TokenLatticeNode">~TokenLatticeNode</a><ul><li><a href="#new_module_utils/data-structures..TokenLatticeNode_new"><code>new TokenLatticeNode(tokenId, nodeId, pos, length, score)</code></a></li> <li><a href="#module_utils/data-structures..TokenLatticeNode+clone"><code>.clone()</code></a> ⇒ <code>TokenLatticeNode</code></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures..TokenLatticeNode_new" class="group"></a> <h3 class="relative group"><a id="new-tokenlatticenodetokenid-nodeid-pos-length-score" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-tokenlatticenodetokenid-nodeid-pos-length-score"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new TokenLatticeNode(tokenId, nodeId, pos, length, score)</span></h3> <p data-svelte-h="svelte-vrhdhn">Represents a node in a token lattice for a given sentence.</p> <table data-svelte-h="svelte-5ahzuq"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>tokenId</td><td><code>number</code></td><td><p>The ID of the token associated with this node.</p></td> </tr><tr><td>nodeId</td><td><code>number</code></td><td><p>The ID of this node.</p></td> </tr><tr><td>pos</td><td><code>number</code></td><td><p>The starting position of the token in the sentence.</p></td> </tr><tr><td>length</td><td><code>number</code></td><td><p>The length of the token.</p></td> </tr><tr><td>score</td><td><code>number</code></td><td><p>The score associated with the token.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures..TokenLatticeNode+clone" class="group"></a> <h3 class="relative group"><a id="tokenlatticenodeclone--code-tokenlatticenode-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticenodeclone--code-tokenlatticenode-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLatticeNode.clone() ⇒ <code> TokenLatticeNode </code></span></h3> <p data-svelte-h="svelte-34g670">Returns a clone of this node.</p> <p data-svelte-h="svelte-7303j9"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures..TokenLatticeNode"><code>TokenLatticeNode</code></a><br> <strong>Returns</strong>: <code>TokenLatticeNode</code> - A clone of this node.</p> <hr> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers.js/blob/main/docs/source/api/utils/data-structures.md" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_kuyevp = {
	assets: "/docs/transformers.js/pr_1113/en",
	base: "/docs/transformers.js/pr_1113/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/transformers.js/pr_1113/en/_app/immutable/entry/start.88a6e140.js"),
	import("/docs/transformers.js/pr_1113/en/_app/immutable/entry/app.0003020d.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 18],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 64.8 kB
Xet hash:: f83b6ac2c13811cf644c062078e2bce7e3c3dc5352c2ee7612a24fd3697459fd

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.