File size: 3,600 Bytes
14a2165 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 |
/* tslint:disable */
/* eslint-disable */
export type TiktokenEncoding = "gpt2" | "r50k_base" | "p50k_base" | "p50k_edit" | "cl100k_base" | "o200k_base";
/**
* @param {TiktokenEncoding} encoding
* @param {Record<string, number>} [extend_special_tokens]
* @returns {Tiktoken}
*/
export function get_encoding(encoding: TiktokenEncoding, extend_special_tokens?: Record<string, number>): Tiktoken;
export type TiktokenModel =
| "davinci-002"
| "babbage-002"
| "text-davinci-003"
| "text-davinci-002"
| "text-davinci-001"
| "text-curie-001"
| "text-babbage-001"
| "text-ada-001"
| "davinci"
| "curie"
| "babbage"
| "ada"
| "code-davinci-002"
| "code-davinci-001"
| "code-cushman-002"
| "code-cushman-001"
| "davinci-codex"
| "cushman-codex"
| "text-davinci-edit-001"
| "code-davinci-edit-001"
| "text-embedding-ada-002"
| "text-similarity-davinci-001"
| "text-similarity-curie-001"
| "text-similarity-babbage-001"
| "text-similarity-ada-001"
| "text-search-davinci-doc-001"
| "text-search-curie-doc-001"
| "text-search-babbage-doc-001"
| "text-search-ada-doc-001"
| "code-search-babbage-code-001"
| "code-search-ada-code-001"
| "gpt2"
| "gpt-3.5-turbo"
| "gpt-35-turbo"
| "gpt-3.5-turbo-0301"
| "gpt-3.5-turbo-0613"
| "gpt-3.5-turbo-1106"
| "gpt-3.5-turbo-0125"
| "gpt-3.5-turbo-16k"
| "gpt-3.5-turbo-16k-0613"
| "gpt-3.5-turbo-instruct"
| "gpt-3.5-turbo-instruct-0914"
| "gpt-4"
| "gpt-4-0314"
| "gpt-4-0613"
| "gpt-4-32k"
| "gpt-4-32k-0314"
| "gpt-4-32k-0613"
| "gpt-4-turbo"
| "gpt-4-turbo-2024-04-09"
| "gpt-4-turbo-preview"
| "gpt-4-1106-preview"
| "gpt-4-0125-preview"
| "gpt-4-vision-preview"
| "gpt-4o"
| "gpt-4o-2024-05-13"
| "gpt-4o-2024-08-06"
| "gpt-4o-mini-2024-07-18"
| "gpt-4o-mini"
| "o1-mini"
| "o1-preview"
| "o1-preview-2024-09-12"
| "o1-mini-2024-09-12"
| "chatgpt-4o-latest"
| "gpt-4o-realtime"
| "gpt-4o-realtime-preview-2024-10-01"
/**
* @param {TiktokenModel} encoding
* @param {Record<string, number>} [extend_special_tokens]
* @returns {Tiktoken}
*/
export function encoding_for_model(model: TiktokenModel, extend_special_tokens?: Record<string, number>): Tiktoken;
/**
*/
export class Tiktoken {
free(): void;
/**
* @param {string} tiktoken_bfe
* @param {any} special_tokens
* @param {string} pat_str
*/
constructor(tiktoken_bfe: string, special_tokens: Record<string, number>, pat_str: string);
/**
* @param {string} text
* @param {any} allowed_special
* @param {any} disallowed_special
* @returns {Uint32Array}
*/
encode(text: string, allowed_special?: "all" | string[], disallowed_special?: "all" | string[]): Uint32Array;
/**
* @param {string} text
* @returns {Uint32Array}
*/
encode_ordinary(text: string): Uint32Array;
/**
* @param {string} text
* @param {any} allowed_special
* @param {any} disallowed_special
* @returns {any}
*/
encode_with_unstable(text: string, allowed_special?: "all" | string[], disallowed_special?: "all" | string[]): any;
/**
* @param {Uint8Array} bytes
* @returns {number}
*/
encode_single_token(bytes: Uint8Array): number;
/**
* @param {Uint32Array} tokens
* @returns {Uint8Array}
*/
decode(tokens: Uint32Array): Uint8Array;
/**
* @param {number} token
* @returns {Uint8Array}
*/
decode_single_token_bytes(token: number): Uint8Array;
/**
* @returns {any}
*/
token_byte_values(): Array<Array<number>>;
/**
*/
readonly name: string | undefined;
}
|