Buckets:

rtrm's picture
download
raw
84.9 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;utils/data-structures&quot;,&quot;local&quot;:&quot;utilsdata-structures&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;utils/data-structures.PriorityQueue&quot;,&quot;local&quot;:&quot;utilsdata-structurespriorityqueue&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new PriorityQueue(comparator)&quot;,&quot;local&quot;:&quot;new-priorityqueuecomparator&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.size&quot;,&quot;local&quot;:&quot;priorityqueuesize&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.isEmpty() ⇒ <code> boolean </code>&quot;,&quot;local&quot;:&quot;priorityqueueisempty--code-boolean-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.peek() ⇒ <code> any </code>&quot;,&quot;local&quot;:&quot;priorityqueuepeek--code-any-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.push(...values) ⇒ <code> number </code>&quot;,&quot;local&quot;:&quot;priorityqueuepushvalues--code-number-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.extend(values) ⇒ <code> number </code>&quot;,&quot;local&quot;:&quot;priorityqueueextendvalues--code-number-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.pop() ⇒ <code> any </code>&quot;,&quot;local&quot;:&quot;priorityqueuepop--code-any-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.replace(value) ⇒ <code> * </code>&quot;,&quot;local&quot;:&quot;priorityqueuereplacevalue--code--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue._siftUpFrom(node)&quot;,&quot;local&quot;:&quot;priorityqueuesiftupfromnode&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures.CharTrie&quot;,&quot;local&quot;:&quot;utilsdata-structureschartrie&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;charTrie.extend(texts)&quot;,&quot;local&quot;:&quot;chartrieextendtexts&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;charTrie.push(text)&quot;,&quot;local&quot;:&quot;chartriepushtext&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;charTrie.commonPrefixSearch(text)&quot;,&quot;local&quot;:&quot;chartriecommonprefixsearchtext&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures.TokenLattice&quot;,&quot;local&quot;:&quot;utilsdata-structurestokenlattice&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new TokenLattice(sentence, bosTokenId, eosTokenId)&quot;,&quot;local&quot;:&quot;new-tokenlatticesentence-bostokenid-eostokenid&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.insert(pos, length, score, tokenId)&quot;,&quot;local&quot;:&quot;tokenlatticeinsertpos-length-score-tokenid&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.viterbi() ⇒ <code> Array. < TokenLatticeNode > </code>&quot;,&quot;local&quot;:&quot;tokenlatticeviterbi--code-array--tokenlatticenode--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.piece(node) ⇒ <code> string </code>&quot;,&quot;local&quot;:&quot;tokenlatticepiecenode--code-string-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.tokens() ⇒ <code> Array. < string > </code>&quot;,&quot;local&quot;:&quot;tokenlatticetokens--code-array--string--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.tokenIds() ⇒ <code> Array. < number > </code>&quot;,&quot;local&quot;:&quot;tokenlatticetokenids--code-array--number--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures.DictionarySplitter&quot;,&quot;local&quot;:&quot;utilsdata-structuresdictionarysplitter&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new DictionarySplitter(dictionary)&quot;,&quot;local&quot;:&quot;new-dictionarysplitterdictionary&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;dictionarySplitter.split(text) ⇒ <code> Array. < string > </code>&quot;,&quot;local&quot;:&quot;dictionarysplittersplittext--code-array--string--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures.LRUCache&quot;,&quot;local&quot;:&quot;utilsdata-structureslrucache&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new LRUCache(capacity)&quot;,&quot;local&quot;:&quot;new-lrucachecapacity&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;lruCache.get(key) ⇒ <code> any </code>&quot;,&quot;local&quot;:&quot;lrucachegetkey--code-any-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;lruCache.put(key, value)&quot;,&quot;local&quot;:&quot;lrucacheputkey-value&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;lruCache.clear()&quot;,&quot;local&quot;:&quot;lrucacheclear&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures~CharTrieNode&quot;,&quot;local&quot;:&quot;utilsdata-structureschartrienode&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new CharTrieNode(isLeaf, children)&quot;,&quot;local&quot;:&quot;new-chartrienodeisleaf-children&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;CharTrieNode.default() ⇒ <code> CharTrieNode </code>&quot;,&quot;local&quot;:&quot;chartrienodedefault--code-chartrienode-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures~TokenLatticeNode&quot;,&quot;local&quot;:&quot;utilsdata-structurestokenlatticenode&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new TokenLatticeNode(tokenId, nodeId, pos, length, score)&quot;,&quot;local&quot;:&quot;new-tokenlatticenodetokenid-nodeid-pos-length-score&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLatticeNode.clone() ⇒ <code> TokenLatticeNode </code>&quot;,&quot;local&quot;:&quot;tokenlatticenodeclone--code-tokenlatticenode-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
<link href="/docs/transformers.js/pr_1472/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/entry/start.8b47b7c3.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/chunks/scheduler.6efaaf90.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/chunks/singletons.f26f3f03.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/chunks/paths.81f7c65a.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/entry/app.880eb531.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/chunks/preload-helper.4ecb51b7.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/chunks/index.eb3e1f0f.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/nodes/0.30e774d6.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/chunks/each.e59479a4.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/nodes/18.cffa2d47.js">
<link rel="modulepreload" href="/docs/transformers.js/pr_1472/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.8570d1c1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;utils/data-structures&quot;,&quot;local&quot;:&quot;utilsdata-structures&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;utils/data-structures.PriorityQueue&quot;,&quot;local&quot;:&quot;utilsdata-structurespriorityqueue&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new PriorityQueue(comparator)&quot;,&quot;local&quot;:&quot;new-priorityqueuecomparator&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.size&quot;,&quot;local&quot;:&quot;priorityqueuesize&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.isEmpty() ⇒ <code> boolean </code>&quot;,&quot;local&quot;:&quot;priorityqueueisempty--code-boolean-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.peek() ⇒ <code> any </code>&quot;,&quot;local&quot;:&quot;priorityqueuepeek--code-any-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.push(...values) ⇒ <code> number </code>&quot;,&quot;local&quot;:&quot;priorityqueuepushvalues--code-number-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.extend(values) ⇒ <code> number </code>&quot;,&quot;local&quot;:&quot;priorityqueueextendvalues--code-number-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.pop() ⇒ <code> any </code>&quot;,&quot;local&quot;:&quot;priorityqueuepop--code-any-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue.replace(value) ⇒ <code> * </code>&quot;,&quot;local&quot;:&quot;priorityqueuereplacevalue--code--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;priorityQueue._siftUpFrom(node)&quot;,&quot;local&quot;:&quot;priorityqueuesiftupfromnode&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures.CharTrie&quot;,&quot;local&quot;:&quot;utilsdata-structureschartrie&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;charTrie.extend(texts)&quot;,&quot;local&quot;:&quot;chartrieextendtexts&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;charTrie.push(text)&quot;,&quot;local&quot;:&quot;chartriepushtext&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;charTrie.commonPrefixSearch(text)&quot;,&quot;local&quot;:&quot;chartriecommonprefixsearchtext&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures.TokenLattice&quot;,&quot;local&quot;:&quot;utilsdata-structurestokenlattice&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new TokenLattice(sentence, bosTokenId, eosTokenId)&quot;,&quot;local&quot;:&quot;new-tokenlatticesentence-bostokenid-eostokenid&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.insert(pos, length, score, tokenId)&quot;,&quot;local&quot;:&quot;tokenlatticeinsertpos-length-score-tokenid&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.viterbi() ⇒ <code> Array. < TokenLatticeNode > </code>&quot;,&quot;local&quot;:&quot;tokenlatticeviterbi--code-array--tokenlatticenode--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.piece(node) ⇒ <code> string </code>&quot;,&quot;local&quot;:&quot;tokenlatticepiecenode--code-string-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.tokens() ⇒ <code> Array. < string > </code>&quot;,&quot;local&quot;:&quot;tokenlatticetokens--code-array--string--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLattice.tokenIds() ⇒ <code> Array. < number > </code>&quot;,&quot;local&quot;:&quot;tokenlatticetokenids--code-array--number--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures.DictionarySplitter&quot;,&quot;local&quot;:&quot;utilsdata-structuresdictionarysplitter&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new DictionarySplitter(dictionary)&quot;,&quot;local&quot;:&quot;new-dictionarysplitterdictionary&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;dictionarySplitter.split(text) ⇒ <code> Array. < string > </code>&quot;,&quot;local&quot;:&quot;dictionarysplittersplittext--code-array--string--code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures.LRUCache&quot;,&quot;local&quot;:&quot;utilsdata-structureslrucache&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new LRUCache(capacity)&quot;,&quot;local&quot;:&quot;new-lrucachecapacity&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;lruCache.get(key) ⇒ <code> any </code>&quot;,&quot;local&quot;:&quot;lrucachegetkey--code-any-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;lruCache.put(key, value)&quot;,&quot;local&quot;:&quot;lrucacheputkey-value&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;lruCache.clear()&quot;,&quot;local&quot;:&quot;lrucacheclear&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures~CharTrieNode&quot;,&quot;local&quot;:&quot;utilsdata-structureschartrienode&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new CharTrieNode(isLeaf, children)&quot;,&quot;local&quot;:&quot;new-chartrienodeisleaf-children&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;CharTrieNode.default() ⇒ <code> CharTrieNode </code>&quot;,&quot;local&quot;:&quot;chartrienodedefault--code-chartrienode-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;utils/data-structures~TokenLatticeNode&quot;,&quot;local&quot;:&quot;utilsdata-structurestokenlatticenode&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;new TokenLatticeNode(tokenId, nodeId, pos, length, score)&quot;,&quot;local&quot;:&quot;new-tokenlatticenodetokenid-nodeid-pos-length-score&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;tokenLatticeNode.clone() ⇒ <code> TokenLatticeNode </code>&quot;,&quot;local&quot;:&quot;tokenlatticenodeclone--code-tokenlatticenode-code&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <a id="module_utils/data-structures" class="group"></a> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 max-sm:gap-0.5 h-6 max-sm:h-5 px-2 max-sm:px-1.5 text-[11px] max-sm:text-[9px] font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0"><svg class="w-3 h-3 max-sm:w-2.5 max-sm:h-2.5" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-6 max-sm:h-5 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible w-3 h-3 max-sm:w-2.5 max-sm:h-2.5 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="utilsdata-structures" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structures"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures</span></h1> <p data-svelte-h="svelte-1aeeeid">Custom data structures.</p> <p data-svelte-h="svelte-1r3ntfo">These are only used internally, meaning an end-user shouldn’t
need to access anything here.</p> <ul data-svelte-h="svelte-1m3xm6o"><li><a href="#module_utils/data-structures">utils/data-structures</a> <ul><li><em>static</em> <ul><li><a href="#module_utils/data-structures.PriorityQueue">.PriorityQueue</a> <ul><li><a href="#new_module_utils/data-structures.PriorityQueue_new"><code>new PriorityQueue(comparator)</code></a></li> <li><a href="#module_utils/data-structures.PriorityQueue+size"><code>.size</code></a></li> <li><a href="#module_utils/data-structures.PriorityQueue+isEmpty"><code>.isEmpty()</code></a><code>boolean</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+peek"><code>.peek()</code></a><code>any</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+push"><code>.push(...values)</code></a><code>number</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+extend"><code>.extend(values)</code></a><code>number</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+pop"><code>.pop()</code></a><code>any</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+replace"><code>.replace(value)</code></a><code>*</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+_siftUpFrom"><code>._siftUpFrom(node)</code></a></li></ul></li> <li><a href="#module_utils/data-structures.CharTrie">.CharTrie</a> <ul><li><a href="#module_utils/data-structures.CharTrie+extend"><code>.extend(texts)</code></a></li> <li><a href="#module_utils/data-structures.CharTrie+push"><code>.push(text)</code></a></li> <li><a href="#module_utils/data-structures.CharTrie+commonPrefixSearch"><code>.commonPrefixSearch(text)</code></a></li></ul></li> <li><a href="#module_utils/data-structures.TokenLattice">.TokenLattice</a> <ul><li><a href="#new_module_utils/data-structures.TokenLattice_new"><code>new TokenLattice(sentence, bosTokenId, eosTokenId)</code></a></li> <li><a href="#module_utils/data-structures.TokenLattice+insert"><code>.insert(pos, length, score, tokenId)</code></a></li> <li><a href="#module_utils/data-structures.TokenLattice+viterbi"><code>.viterbi()</code></a><code>Array.&lt;TokenLatticeNode&gt;</code></li> <li><a href="#module_utils/data-structures.TokenLattice+piece"><code>.piece(node)</code></a><code>string</code></li> <li><a href="#module_utils/data-structures.TokenLattice+tokens"><code>.tokens()</code></a><code>Array.&lt;string&gt;</code></li> <li><a href="#module_utils/data-structures.TokenLattice+tokenIds"><code>.tokenIds()</code></a><code>Array.&lt;number&gt;</code></li></ul></li> <li><a href="#module_utils/data-structures.DictionarySplitter">.DictionarySplitter</a> <ul><li><a href="#new_module_utils/data-structures.DictionarySplitter_new"><code>new DictionarySplitter(dictionary)</code></a></li> <li><a href="#module_utils/data-structures.DictionarySplitter+split"><code>.split(text)</code></a><code>Array.&lt;string&gt;</code></li></ul></li> <li><a href="#module_utils/data-structures.LRUCache">.LRUCache</a> <ul><li><a href="#new_module_utils/data-structures.LRUCache_new"><code>new LRUCache(capacity)</code></a></li> <li><a href="#module_utils/data-structures.LRUCache+get"><code>.get(key)</code></a><code>any</code></li> <li><a href="#module_utils/data-structures.LRUCache+put"><code>.put(key, value)</code></a></li> <li><a href="#module_utils/data-structures.LRUCache+clear"><code>.clear()</code></a></li></ul></li></ul></li> <li><em>inner</em> <ul><li><a href="#module_utils/data-structures..CharTrieNode">~CharTrieNode</a> <ul><li><a href="#new_module_utils/data-structures..CharTrieNode_new"><code>new CharTrieNode(isLeaf, children)</code></a></li> <li><a href="#module_utils/data-structures..CharTrieNode.default"><code>.default()</code></a><code>CharTrieNode</code></li></ul></li> <li><a href="#module_utils/data-structures..TokenLatticeNode">~TokenLatticeNode</a> <ul><li><a href="#new_module_utils/data-structures..TokenLatticeNode_new"><code>new TokenLatticeNode(tokenId, nodeId, pos, length, score)</code></a></li> <li><a href="#module_utils/data-structures..TokenLatticeNode+clone"><code>.clone()</code></a><code>TokenLatticeNode</code></li></ul></li></ul></li></ul></li></ul> <hr> <a id="module_utils/data-structures.PriorityQueue" class="group"></a> <h2 class="relative group"><a id="utilsdata-structurespriorityqueue" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structurespriorityqueue"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures.PriorityQueue</span></h2> <p data-svelte-h="svelte-1ux83uj">Efficient Heap-based Implementation of a Priority Queue.
It uses an array-based binary heap, where the root is at index <code>0</code>, and the
children of node <code>i</code> are located at indices <code>2i + 1</code> and <code>2i + 2</code>, respectively.</p> <p data-svelte-h="svelte-vi2pz9">Adapted from the following sources:</p> <ul data-svelte-h="svelte-qlvgg9"><li><a href="https://stackoverflow.com/a/42919752/13989043" rel="nofollow">https://stackoverflow.com/a/42919752/13989043</a> (original)</li> <li><a href="https://github.com/belladoreai/llama-tokenizer-js" rel="nofollow">https://github.com/belladoreai/llama-tokenizer-js</a> (minor improvements)</li></ul> <p data-svelte-h="svelte-1saulos"><strong>Kind</strong>: static class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-ywxd3x"><li><a href="#module_utils/data-structures.PriorityQueue">.PriorityQueue</a> <ul><li><a href="#new_module_utils/data-structures.PriorityQueue_new"><code>new PriorityQueue(comparator)</code></a></li> <li><a href="#module_utils/data-structures.PriorityQueue+size"><code>.size</code></a></li> <li><a href="#module_utils/data-structures.PriorityQueue+isEmpty"><code>.isEmpty()</code></a><code>boolean</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+peek"><code>.peek()</code></a><code>any</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+push"><code>.push(...values)</code></a><code>number</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+extend"><code>.extend(values)</code></a><code>number</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+pop"><code>.pop()</code></a><code>any</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+replace"><code>.replace(value)</code></a><code>*</code></li> <li><a href="#module_utils/data-structures.PriorityQueue+_siftUpFrom"><code>._siftUpFrom(node)</code></a></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures.PriorityQueue_new" class="group"></a> <h3 class="relative group"><a id="new-priorityqueuecomparator" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-priorityqueuecomparator"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new PriorityQueue(comparator)</span></h3> <p data-svelte-h="svelte-1dgukb8">Create a new PriorityQueue.</p> <table data-svelte-h="svelte-15lsmj7"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>comparator</td><td><code>function</code></td><td><p>Comparator function to determine priority. Defaults to a MaxHeap.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.PriorityQueue+size" class="group"></a> <h3 class="relative group"><a id="priorityqueuesize" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuesize"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.size</span></h3> <p data-svelte-h="svelte-19ll7y9">The size of the queue</p> <p data-svelte-h="svelte-1spompf"><strong>Kind</strong>: instance property of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a></p> <hr> <a id="module_utils/data-structures.PriorityQueue+isEmpty" class="group"></a> <h3 class="relative group"><a id="priorityqueueisempty--code-boolean-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueueisempty--code-boolean-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.isEmpty() ⇒ &lt;code> boolean &lt;/code></span></h3> <p data-svelte-h="svelte-iwo0ja">Check if the queue is empty.</p> <p data-svelte-h="svelte-usw3zo"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>boolean</code> - <code>true</code> if the queue is empty, <code>false</code> otherwise.</p> <hr> <a id="module_utils/data-structures.PriorityQueue+peek" class="group"></a> <h3 class="relative group"><a id="priorityqueuepeek--code-any-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuepeek--code-any-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.peek() ⇒ &lt;code> any &lt;/code></span></h3> <p data-svelte-h="svelte-1g38rzl">Return the element with the highest priority in the queue.</p> <p data-svelte-h="svelte-1jn2os3"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>any</code> - The highest priority element in the queue.</p> <hr> <a id="module_utils/data-structures.PriorityQueue+push" class="group"></a> <h3 class="relative group"><a id="priorityqueuepushvalues--code-number-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuepushvalues--code-number-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.push(...values) ⇒ &lt;code> number &lt;/code></span></h3> <p data-svelte-h="svelte-1o3p42d">Add one or more elements to the queue.</p> <p data-svelte-h="svelte-1hty3q7"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>number</code> - The new size of the queue.</p> <table data-svelte-h="svelte-19plha1"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>...values</td><td><code>any</code></td><td><p>The values to push into the queue.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.PriorityQueue+extend" class="group"></a> <h3 class="relative group"><a id="priorityqueueextendvalues--code-number-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueueextendvalues--code-number-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.extend(values) ⇒ &lt;code> number &lt;/code></span></h3> <p data-svelte-h="svelte-1rf8gmz">Add multiple elements to the queue.</p> <p data-svelte-h="svelte-1hty3q7"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>number</code> - The new size of the queue.</p> <table data-svelte-h="svelte-txh99d"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>values</td><td><code>Array.&lt;any&gt;</code></td><td><p>The values to push into the queue.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.PriorityQueue+pop" class="group"></a> <h3 class="relative group"><a id="priorityqueuepop--code-any-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuepop--code-any-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.pop() ⇒ &lt;code> any &lt;/code></span></h3> <p data-svelte-h="svelte-1l3oksc">Remove and return the element with the highest priority in the queue.</p> <p data-svelte-h="svelte-vpswb8"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>any</code> - The element with the highest priority in the queue.</p> <hr> <a id="module_utils/data-structures.PriorityQueue+replace" class="group"></a> <h3 class="relative group"><a id="priorityqueuereplacevalue--code--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuereplacevalue--code--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue.replace(value) ⇒ &lt;code> * &lt;/code></span></h3> <p data-svelte-h="svelte-7s4t0d">Replace the element with the highest priority in the queue with a new value.</p> <p data-svelte-h="svelte-ckytvb"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a><br> <strong>Returns</strong>: <code>*</code> - The replaced value.</p> <table data-svelte-h="svelte-1xv2m9q"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>value</td><td><code>*</code></td><td><p>The new value.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.PriorityQueue+_siftUpFrom" class="group"></a> <h3 class="relative group"><a id="priorityqueuesiftupfromnode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#priorityqueuesiftupfromnode"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>priorityQueue._siftUpFrom(node)</span></h3> <p data-svelte-h="svelte-hj5sse">Helper function to sift up from a given node.</p> <p data-svelte-h="svelte-6p6lmz"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.PriorityQueue"><code>PriorityQueue</code></a></p> <table data-svelte-h="svelte-fj2fu5"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>node</td><td><code>number</code></td><td><p>The index of the node to start sifting up from.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.CharTrie" class="group"></a> <h2 class="relative group"><a id="utilsdata-structureschartrie" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structureschartrie"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures.CharTrie</span></h2> <p data-svelte-h="svelte-nklgrc">A trie structure to efficiently store and search for strings.</p> <p data-svelte-h="svelte-1saulos"><strong>Kind</strong>: static class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-ktb0g6"><li><a href="#module_utils/data-structures.CharTrie">.CharTrie</a> <ul><li><a href="#module_utils/data-structures.CharTrie+extend"><code>.extend(texts)</code></a></li> <li><a href="#module_utils/data-structures.CharTrie+push"><code>.push(text)</code></a></li> <li><a href="#module_utils/data-structures.CharTrie+commonPrefixSearch"><code>.commonPrefixSearch(text)</code></a></li></ul></li></ul> <hr> <a id="module_utils/data-structures.CharTrie+extend" class="group"></a> <h3 class="relative group"><a id="chartrieextendtexts" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#chartrieextendtexts"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>charTrie.extend(texts)</span></h3> <p data-svelte-h="svelte-moeq8l">Adds one or more <code>texts</code> to the trie.</p> <p data-svelte-h="svelte-33rax3"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.CharTrie"><code>CharTrie</code></a></p> <table data-svelte-h="svelte-18dtsnv"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>texts</td><td><code>Array.&lt;string&gt;</code></td><td><p>The strings to add to the trie.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.CharTrie+push" class="group"></a> <h3 class="relative group"><a id="chartriepushtext" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#chartriepushtext"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>charTrie.push(text)</span></h3> <p data-svelte-h="svelte-4q846v">Adds text to the trie.</p> <p data-svelte-h="svelte-33rax3"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.CharTrie"><code>CharTrie</code></a></p> <table data-svelte-h="svelte-hqwgb9"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>text</td><td><code>string</code></td><td><p>The string to add to the trie.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.CharTrie+commonPrefixSearch" class="group"></a> <h3 class="relative group"><a id="chartriecommonprefixsearchtext" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#chartriecommonprefixsearchtext"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>charTrie.commonPrefixSearch(text)</span></h3> <p data-svelte-h="svelte-bmaw98">Searches the trie for all strings with a common prefix of <code>text</code>.</p> <p data-svelte-h="svelte-33rax3"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.CharTrie"><code>CharTrie</code></a></p> <table data-svelte-h="svelte-18qxh0x"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>text</td><td><code>string</code></td><td><p>The common prefix to search for.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.TokenLattice" class="group"></a> <h2 class="relative group"><a id="utilsdata-structurestokenlattice" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structurestokenlattice"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures.TokenLattice</span></h2> <p data-svelte-h="svelte-1xkinx1">A lattice data structure to be used for tokenization.</p> <p data-svelte-h="svelte-1saulos"><strong>Kind</strong>: static class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-1ubeqv9"><li><a href="#module_utils/data-structures.TokenLattice">.TokenLattice</a> <ul><li><a href="#new_module_utils/data-structures.TokenLattice_new"><code>new TokenLattice(sentence, bosTokenId, eosTokenId)</code></a></li> <li><a href="#module_utils/data-structures.TokenLattice+insert"><code>.insert(pos, length, score, tokenId)</code></a></li> <li><a href="#module_utils/data-structures.TokenLattice+viterbi"><code>.viterbi()</code></a><code>Array.&lt;TokenLatticeNode&gt;</code></li> <li><a href="#module_utils/data-structures.TokenLattice+piece"><code>.piece(node)</code></a><code>string</code></li> <li><a href="#module_utils/data-structures.TokenLattice+tokens"><code>.tokens()</code></a><code>Array.&lt;string&gt;</code></li> <li><a href="#module_utils/data-structures.TokenLattice+tokenIds"><code>.tokenIds()</code></a><code>Array.&lt;number&gt;</code></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures.TokenLattice_new" class="group"></a> <h3 class="relative group"><a id="new-tokenlatticesentence-bostokenid-eostokenid" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-tokenlatticesentence-bostokenid-eostokenid"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new TokenLattice(sentence, bosTokenId, eosTokenId)</span></h3> <p data-svelte-h="svelte-ziiz62">Creates a new TokenLattice instance.</p> <table data-svelte-h="svelte-1h8dv5n"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>sentence</td><td><code>string</code></td><td><p>The input sentence to be tokenized.</p></td> </tr><tr><td>bosTokenId</td><td><code>number</code></td><td><p>The beginning-of-sequence token ID.</p></td> </tr><tr><td>eosTokenId</td><td><code>number</code></td><td><p>The end-of-sequence token ID.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.TokenLattice+insert" class="group"></a> <h3 class="relative group"><a id="tokenlatticeinsertpos-length-score-tokenid" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticeinsertpos-length-score-tokenid"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.insert(pos, length, score, tokenId)</span></h3> <p data-svelte-h="svelte-1hzq3te">Inserts a new token node into the token lattice.</p> <p data-svelte-h="svelte-1bjyh7n"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a></p> <table data-svelte-h="svelte-1a01i2a"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>pos</td><td><code>number</code></td><td><p>The starting position of the token.</p></td> </tr><tr><td>length</td><td><code>number</code></td><td><p>The length of the token.</p></td> </tr><tr><td>score</td><td><code>number</code></td><td><p>The score of the token.</p></td> </tr><tr><td>tokenId</td><td><code>number</code></td><td><p>The token ID of the token.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.TokenLattice+viterbi" class="group"></a> <h3 class="relative group"><a id="tokenlatticeviterbi--code-array--tokenlatticenode--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticeviterbi--code-array--tokenlatticenode--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.viterbi() ⇒ &lt;code> Array. &lt; TokenLatticeNode > &lt;/code></span></h3> <p data-svelte-h="svelte-jv0r">Implements the Viterbi algorithm to compute the most likely sequence of tokens.</p> <p data-svelte-h="svelte-1hp113b"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a><br> <strong>Returns</strong>: <code>Array.&lt;TokenLatticeNode&gt;</code> - The most likely sequence of tokens.</p> <hr> <a id="module_utils/data-structures.TokenLattice+piece" class="group"></a> <h3 class="relative group"><a id="tokenlatticepiecenode--code-string-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticepiecenode--code-string-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.piece(node) ⇒ &lt;code> string &lt;/code></span></h3> <p data-svelte-h="svelte-1n9zjoi"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a><br> <strong>Returns</strong>: <code>string</code> - The array of nodes representing the most likely sequence of tokens.</p> <table data-svelte-h="svelte-6nufyb"><thead><tr><th>Param</th><th>Type</th></tr></thead> <tbody><tr><td>node</td><td><code>TokenLatticeNode</code></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.TokenLattice+tokens" class="group"></a> <h3 class="relative group"><a id="tokenlatticetokens--code-array--string--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticetokens--code-array--string--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.tokens() ⇒ &lt;code> Array. &lt; string > &lt;/code></span></h3> <p data-svelte-h="svelte-1ipxykz"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a><br> <strong>Returns</strong>: <code>Array.&lt;string&gt;</code> - The most likely sequence of tokens.</p> <hr> <a id="module_utils/data-structures.TokenLattice+tokenIds" class="group"></a> <h3 class="relative group"><a id="tokenlatticetokenids--code-array--number--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticetokenids--code-array--number--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLattice.tokenIds() ⇒ &lt;code> Array. &lt; number > &lt;/code></span></h3> <p data-svelte-h="svelte-1nolezc"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.TokenLattice"><code>TokenLattice</code></a><br> <strong>Returns</strong>: <code>Array.&lt;number&gt;</code> - The most likely sequence of token ids.</p> <hr> <a id="module_utils/data-structures.DictionarySplitter" class="group"></a> <h2 class="relative group"><a id="utilsdata-structuresdictionarysplitter" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structuresdictionarysplitter"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures.DictionarySplitter</span></h2> <p data-svelte-h="svelte-1d71m1c">A data structure which uses a trie to split a string into tokens based on a dictionary.
It can also use a regular expression to preprocess the input text before splitting.</p> <p data-svelte-h="svelte-13fkdt9">NOTE: To ensure multi-byte characters are handled correctly, we operate at byte-level instead of character-level.</p> <p data-svelte-h="svelte-1saulos"><strong>Kind</strong>: static class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-1glduhs"><li><a href="#module_utils/data-structures.DictionarySplitter">.DictionarySplitter</a> <ul><li><a href="#new_module_utils/data-structures.DictionarySplitter_new"><code>new DictionarySplitter(dictionary)</code></a></li> <li><a href="#module_utils/data-structures.DictionarySplitter+split"><code>.split(text)</code></a><code>Array.&lt;string&gt;</code></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures.DictionarySplitter_new" class="group"></a> <h3 class="relative group"><a id="new-dictionarysplitterdictionary" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-dictionarysplitterdictionary"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new DictionarySplitter(dictionary)</span></h3> <table data-svelte-h="svelte-d113ps"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>dictionary</td><td><code>Array.&lt;string&gt;</code></td><td><p>The dictionary of words to use for splitting.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.DictionarySplitter+split" class="group"></a> <h3 class="relative group"><a id="dictionarysplittersplittext--code-array--string--code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dictionarysplittersplittext--code-array--string--code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>dictionarySplitter.split(text) ⇒ &lt;code> Array. &lt; string > &lt;/code></span></h3> <p data-svelte-h="svelte-ussbd6">Splits the input text into tokens based on the dictionary.</p> <p data-svelte-h="svelte-kdhwpe"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.DictionarySplitter"><code>DictionarySplitter</code></a><br> <strong>Returns</strong>: <code>Array.&lt;string&gt;</code> - An array of tokens.</p> <table data-svelte-h="svelte-raiep6"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>text</td><td><code>string</code></td><td><p>The input text to split.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.LRUCache" class="group"></a> <h2 class="relative group"><a id="utilsdata-structureslrucache" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structureslrucache"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures.LRUCache</span></h2> <p data-svelte-h="svelte-19p0c6a">A simple Least Recently Used (LRU) cache implementation in JavaScript.
This cache stores key-value pairs and evicts the least recently used item
when the capacity is exceeded.</p> <p data-svelte-h="svelte-1saulos"><strong>Kind</strong>: static class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-1i8ipgs"><li><a href="#module_utils/data-structures.LRUCache">.LRUCache</a> <ul><li><a href="#new_module_utils/data-structures.LRUCache_new"><code>new LRUCache(capacity)</code></a></li> <li><a href="#module_utils/data-structures.LRUCache+get"><code>.get(key)</code></a><code>any</code></li> <li><a href="#module_utils/data-structures.LRUCache+put"><code>.put(key, value)</code></a></li> <li><a href="#module_utils/data-structures.LRUCache+clear"><code>.clear()</code></a></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures.LRUCache_new" class="group"></a> <h3 class="relative group"><a id="new-lrucachecapacity" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-lrucachecapacity"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new LRUCache(capacity)</span></h3> <p data-svelte-h="svelte-1wrhlc6">Creates an LRUCache instance.</p> <table data-svelte-h="svelte-1puj8on"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>capacity</td><td><code>number</code></td><td><p>The maximum number of items the cache can hold.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.LRUCache+get" class="group"></a> <h3 class="relative group"><a id="lrucachegetkey--code-any-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#lrucachegetkey--code-any-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>lruCache.get(key) ⇒ &lt;code> any &lt;/code></span></h3> <p data-svelte-h="svelte-u7qnnu">Retrieves the value associated with the given key and marks the key as recently used.</p> <p data-svelte-h="svelte-15ir7qc"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.LRUCache"><code>LRUCache</code></a><br> <strong>Returns</strong>: <code>any</code> - The value associated with the key, or undefined if the key does not exist.</p> <table data-svelte-h="svelte-7pa8k3"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>key</td><td><code>any</code></td><td><p>The key to retrieve.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.LRUCache+put" class="group"></a> <h3 class="relative group"><a id="lrucacheputkey-value" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#lrucacheputkey-value"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>lruCache.put(key, value)</span></h3> <p data-svelte-h="svelte-13y3987">Inserts or updates the key-value pair in the cache.
If the key already exists, it is updated and marked as recently used.
If the cache exceeds its capacity, the least recently used item is evicted.</p> <p data-svelte-h="svelte-134bybf"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.LRUCache"><code>LRUCache</code></a></p> <table data-svelte-h="svelte-1urvam6"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>key</td><td><code>any</code></td><td><p>The key to add or update.</p></td> </tr><tr><td>value</td><td><code>any</code></td><td><p>The value to associate with the key.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures.LRUCache+clear" class="group"></a> <h3 class="relative group"><a id="lrucacheclear" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#lrucacheclear"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>lruCache.clear()</span></h3> <p data-svelte-h="svelte-1rw967f">Clears the cache.</p> <p data-svelte-h="svelte-134bybf"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures.LRUCache"><code>LRUCache</code></a></p> <hr> <a id="module_utils/data-structures..CharTrieNode" class="group"></a> <h2 class="relative group"><a id="utilsdata-structureschartrienode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structureschartrienode"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures~CharTrieNode</span></h2> <p data-svelte-h="svelte-1auz3rh">Represents a node in a character trie.</p> <p data-svelte-h="svelte-1p9ver0"><strong>Kind</strong>: inner class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-1ysrfb6"><li><a href="#module_utils/data-structures..CharTrieNode">~CharTrieNode</a> <ul><li><a href="#new_module_utils/data-structures..CharTrieNode_new"><code>new CharTrieNode(isLeaf, children)</code></a></li> <li><a href="#module_utils/data-structures..CharTrieNode.default"><code>.default()</code></a><code>CharTrieNode</code></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures..CharTrieNode_new" class="group"></a> <h3 class="relative group"><a id="new-chartrienodeisleaf-children" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-chartrienodeisleaf-children"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new CharTrieNode(isLeaf, children)</span></h3> <p data-svelte-h="svelte-mt1jex">Create a new CharTrieNode.</p> <table data-svelte-h="svelte-nk427c"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>isLeaf</td><td><code>boolean</code></td><td><p>Whether the node is a leaf node or not.</p></td> </tr><tr><td>children</td><td><code>Map.&lt;string, CharTrieNode&gt;</code></td><td><p>A map containing the node&#39;s children, where the key is a character and the value is a <code>CharTrieNode</code>.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures..CharTrieNode.default" class="group"></a> <h3 class="relative group"><a id="chartrienodedefault--code-chartrienode-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#chartrienodedefault--code-chartrienode-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>CharTrieNode.default() ⇒ &lt;code> CharTrieNode &lt;/code></span></h3> <p data-svelte-h="svelte-9cjdkj">Returns a new <code>CharTrieNode</code> instance with default values.</p> <p data-svelte-h="svelte-3pigfy"><strong>Kind</strong>: static method of <a href="#module_utils/data-structures..CharTrieNode"><code>CharTrieNode</code></a><br> <strong>Returns</strong>: <code>CharTrieNode</code> - A new <code>CharTrieNode</code> instance with <code>isLeaf</code> set to <code>false</code> and an empty <code>children</code> map.</p> <hr> <a id="module_utils/data-structures..TokenLatticeNode" class="group"></a> <h2 class="relative group"><a id="utilsdata-structurestokenlatticenode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#utilsdata-structurestokenlatticenode"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>utils/data-structures~TokenLatticeNode</span></h2> <p data-svelte-h="svelte-1p9ver0"><strong>Kind</strong>: inner class of <a href="#module_utils/data-structures"><code>utils/data-structures</code></a></p> <ul data-svelte-h="svelte-1yr6rfl"><li><a href="#module_utils/data-structures..TokenLatticeNode">~TokenLatticeNode</a> <ul><li><a href="#new_module_utils/data-structures..TokenLatticeNode_new"><code>new TokenLatticeNode(tokenId, nodeId, pos, length, score)</code></a></li> <li><a href="#module_utils/data-structures..TokenLatticeNode+clone"><code>.clone()</code></a><code>TokenLatticeNode</code></li></ul></li></ul> <hr> <a id="new_module_utils/data-structures..TokenLatticeNode_new" class="group"></a> <h3 class="relative group"><a id="new-tokenlatticenodetokenid-nodeid-pos-length-score" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#new-tokenlatticenodetokenid-nodeid-pos-length-score"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>new TokenLatticeNode(tokenId, nodeId, pos, length, score)</span></h3> <p data-svelte-h="svelte-vrhdhn">Represents a node in a token lattice for a given sentence.</p> <table data-svelte-h="svelte-5ahzuq"><thead><tr><th>Param</th><th>Type</th><th>Description</th></tr></thead> <tbody><tr><td>tokenId</td><td><code>number</code></td><td><p>The ID of the token associated with this node.</p></td> </tr><tr><td>nodeId</td><td><code>number</code></td><td><p>The ID of this node.</p></td> </tr><tr><td>pos</td><td><code>number</code></td><td><p>The starting position of the token in the sentence.</p></td> </tr><tr><td>length</td><td><code>number</code></td><td><p>The length of the token.</p></td> </tr><tr><td>score</td><td><code>number</code></td><td><p>The score associated with the token.</p></td></tr></tbody></table> <hr> <a id="module_utils/data-structures..TokenLatticeNode+clone" class="group"></a> <h3 class="relative group"><a id="tokenlatticenodeclone--code-tokenlatticenode-code" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenlatticenodeclone--code-tokenlatticenode-code"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>tokenLatticeNode.clone() ⇒ &lt;code> TokenLatticeNode &lt;/code></span></h3> <p data-svelte-h="svelte-34g670">Returns a clone of this node.</p> <p data-svelte-h="svelte-7303j9"><strong>Kind</strong>: instance method of <a href="#module_utils/data-structures..TokenLatticeNode"><code>TokenLatticeNode</code></a><br> <strong>Returns</strong>: <code>TokenLatticeNode</code> - A clone of this node.</p> <hr> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers.js/blob/main/docs/source/api/utils/data-structures.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>
<script>
{
__sveltekit_88dkpc = {
assets: "/docs/transformers.js/pr_1472/en",
base: "/docs/transformers.js/pr_1472/en",
env: {}
};
const element = document.currentScript.parentElement;
const data = [null,null];
Promise.all([
import("/docs/transformers.js/pr_1472/en/_app/immutable/entry/start.8b47b7c3.js"),
import("/docs/transformers.js/pr_1472/en/_app/immutable/entry/app.880eb531.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 18],
data,
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
84.9 kB
·
Xet hash:
ce0e6976270eabbdc6fd62d613f69f1f2b18ac451d67197eec72812aec08ce0d

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.